feat(practice): run realtime practice on App-Plus through renderjs
App-Plus has no WebRTC or getUserMedia in the logic layer, so realtime practice was H5-only and the native entry had to advertise a record-then- transcribe downgrade. Move the media/WebRTC engine into `realtime-browser-engine.js` and drive it from `RealtimePracticeAppBridge.vue`, whose renderjs script runs inside the system WebView where those APIs do exist. The logic layer keeps the authenticated calls: SDP exchange and `search_knowledge` tool invocation stay server-proxied, so the access token is never handed to the view layer. Both sides talk over the renderjs bridge only in session payloads the server already assembled. Verified against a real App build (`app-renderjs.js` holds RTCPeerConnection and getUserMedia; `app-service.js` holds the SDP endpoint, renderjs side has zero hits). Microphone permission, live transcription, remote playback, foreground/background switching and disconnect fallback still require Android and iOS device verification. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,220 @@
|
||||
<template>
|
||||
<view
|
||||
class="realtime-app-bridge"
|
||||
:bridge-command="bridgeCommand"
|
||||
:change:bridge-command="realtimeBridge.onCommand"
|
||||
/>
|
||||
</template>
|
||||
|
||||
<script>
|
||||
import { exchangeRealtimeSdp, invokeRealtimeTool } from '@/services/realtime-practice';
|
||||
|
||||
const errorMessage = (error, fallback) =>
|
||||
error instanceof Error ? error.message : fallback;
|
||||
|
||||
export default {
|
||||
emits: [
|
||||
'status',
|
||||
'trainee-transcript',
|
||||
'customer-transcript',
|
||||
'audio-playback-blocked',
|
||||
'terminated',
|
||||
'error'
|
||||
],
|
||||
data() {
|
||||
return {
|
||||
bridgeCommand: { id: 0, type: 'idle' },
|
||||
commandId: 0,
|
||||
activeSessionId: '',
|
||||
pendingStart: null,
|
||||
pendingResume: new Map()
|
||||
};
|
||||
},
|
||||
methods: {
|
||||
sendCommand(type, payload = {}) {
|
||||
this.commandId += 1;
|
||||
this.bridgeCommand = { id: this.commandId, type, ...payload };
|
||||
},
|
||||
start(sessionUpdate, sessionId) {
|
||||
this.pendingStart?.resolve();
|
||||
this.activeSessionId = sessionId || '';
|
||||
return new Promise((resolve, reject) => {
|
||||
this.pendingStart = { resolve, reject };
|
||||
this.sendCommand('start', { sessionUpdate, sessionId: this.activeSessionId });
|
||||
});
|
||||
},
|
||||
stop() {
|
||||
this.pendingStart?.resolve();
|
||||
this.pendingStart = null;
|
||||
this.pendingResume.forEach((resolve) => resolve(false));
|
||||
this.pendingResume.clear();
|
||||
this.activeSessionId = '';
|
||||
this.sendCommand('stop');
|
||||
},
|
||||
resumeAudio() {
|
||||
const requestId = `resume_${Date.now()}_${this.commandId + 1}`;
|
||||
return new Promise((resolve) => {
|
||||
this.pendingResume.set(requestId, resolve);
|
||||
this.sendCommand('resume-audio', { requestId });
|
||||
});
|
||||
},
|
||||
async answerSdp(event) {
|
||||
const sessionId = this.activeSessionId;
|
||||
try {
|
||||
const answerSdp = await exchangeRealtimeSdp(event.offerSdp, sessionId);
|
||||
this.sendCommand('request-result', { requestId: event.requestId, value: answerSdp });
|
||||
} catch (error) {
|
||||
this.sendCommand('request-error', {
|
||||
requestId: event.requestId,
|
||||
message: errorMessage(error, '实时语音连接请求失败')
|
||||
});
|
||||
}
|
||||
},
|
||||
async answerToolCall(event) {
|
||||
const sessionId = this.activeSessionId;
|
||||
try {
|
||||
const response = await invokeRealtimeTool(
|
||||
sessionId,
|
||||
event.callId,
|
||||
event.name,
|
||||
event.argsJson
|
||||
);
|
||||
this.sendCommand('request-result', { requestId: event.requestId, value: response.output });
|
||||
} catch (error) {
|
||||
this.sendCommand('request-error', {
|
||||
requestId: event.requestId,
|
||||
message: errorMessage(error, '实时工具调用失败')
|
||||
});
|
||||
}
|
||||
},
|
||||
onBridgeEvent(event) {
|
||||
if (!event?.type) return;
|
||||
if (event.type === 'sdp-offer') {
|
||||
void this.answerSdp(event);
|
||||
return;
|
||||
}
|
||||
if (event.type === 'tool-call') {
|
||||
void this.answerToolCall(event);
|
||||
return;
|
||||
}
|
||||
if (event.type === 'ready') {
|
||||
this.pendingStart?.resolve();
|
||||
this.pendingStart = null;
|
||||
return;
|
||||
}
|
||||
if (event.type === 'fatal-error') {
|
||||
this.pendingStart?.reject(new Error(event.message || '实时语音启动失败'));
|
||||
this.pendingStart = null;
|
||||
return;
|
||||
}
|
||||
if (event.type === 'resume-result') {
|
||||
this.pendingResume.get(event.requestId)?.(Boolean(event.success));
|
||||
this.pendingResume.delete(event.requestId);
|
||||
return;
|
||||
}
|
||||
if (event.type === 'status') this.$emit('status', event.status);
|
||||
if (event.type === 'trainee-transcript') this.$emit('trainee-transcript', event.text);
|
||||
if (event.type === 'customer-transcript') this.$emit('customer-transcript', event.text);
|
||||
if (event.type === 'audio-playback-blocked') this.$emit('audio-playback-blocked', Boolean(event.blocked));
|
||||
if (event.type === 'terminated') this.$emit('terminated');
|
||||
if (event.type === 'error') this.$emit('error', event.message);
|
||||
}
|
||||
}
|
||||
};
|
||||
</script>
|
||||
|
||||
<script module="realtimeBridge" lang="renderjs">
|
||||
import { createRealtimeBrowserEngine } from '../../services/realtime-browser-engine.js';
|
||||
|
||||
export default {
|
||||
mounted() {
|
||||
this.ownerInstance = this.$ownerInstance;
|
||||
this.pendingRequests = new Map();
|
||||
this.requestSequence = 0;
|
||||
this.startGeneration = 0;
|
||||
},
|
||||
methods: {
|
||||
emit(type, payload = {}) {
|
||||
const owner = this.ownerInstance || this.$ownerInstance;
|
||||
owner?.callMethod('onBridgeEvent', { type, ...payload });
|
||||
},
|
||||
requestLogic(type, payload) {
|
||||
this.requestSequence += 1;
|
||||
const requestId = `render_${Date.now()}_${this.requestSequence}`;
|
||||
return new Promise((resolve, reject) => {
|
||||
this.pendingRequests.set(requestId, { resolve, reject });
|
||||
this.emit(type, { requestId, ...payload });
|
||||
});
|
||||
},
|
||||
ensureEngine() {
|
||||
if (this.engine) return;
|
||||
this.engine = createRealtimeBrowserEngine({
|
||||
onStatus: (status) => this.emit('status', { status }),
|
||||
onTraineeTranscript: (text) => this.emit('trainee-transcript', { text }),
|
||||
onCustomerTranscript: (text) => this.emit('customer-transcript', { text }),
|
||||
onAudioPlaybackBlocked: (blocked) => this.emit('audio-playback-blocked', { blocked }),
|
||||
onToolCall: (callId, name, argsJson) =>
|
||||
this.requestLogic('tool-call', { callId, name, argsJson }),
|
||||
onTerminated: () => this.emit('terminated'),
|
||||
onError: (message) => this.emit('error', { message })
|
||||
}, (offerSdp, sessionId) =>
|
||||
this.requestLogic('sdp-offer', { offerSdp, sessionId: sessionId || '' }));
|
||||
},
|
||||
settleRequest(command) {
|
||||
const pending = this.pendingRequests.get(command.requestId);
|
||||
if (!pending) return;
|
||||
this.pendingRequests.delete(command.requestId);
|
||||
if (command.type === 'request-error') {
|
||||
pending.reject(new Error(command.message || '实时语音请求失败'));
|
||||
} else {
|
||||
pending.resolve(command.value);
|
||||
}
|
||||
},
|
||||
cancelRequests() {
|
||||
this.pendingRequests.forEach(({ reject }) => reject(new Error('实时陪练已结束')));
|
||||
this.pendingRequests.clear();
|
||||
},
|
||||
onCommand(command, previous, ownerInstance) {
|
||||
if (!command?.id || command.id === previous?.id) return;
|
||||
this.ownerInstance = ownerInstance || this.ownerInstance || this.$ownerInstance;
|
||||
if (command.type === 'request-result' || command.type === 'request-error') {
|
||||
this.settleRequest(command);
|
||||
return;
|
||||
}
|
||||
this.ensureEngine();
|
||||
if (command.type === 'stop') {
|
||||
this.startGeneration += 1;
|
||||
this.cancelRequests();
|
||||
this.engine.stop();
|
||||
return;
|
||||
}
|
||||
if (command.type === 'resume-audio') {
|
||||
void this.engine.resumeAudio()
|
||||
.then((success) => this.emit('resume-result', { requestId: command.requestId, success }))
|
||||
.catch(() => this.emit('resume-result', { requestId: command.requestId, success: false }));
|
||||
return;
|
||||
}
|
||||
if (command.type !== 'start') return;
|
||||
const generation = ++this.startGeneration;
|
||||
void this.engine.start(command.sessionUpdate || {}, command.sessionId || '')
|
||||
.then(() => {
|
||||
if (generation === this.startGeneration) this.emit('ready');
|
||||
})
|
||||
.catch((error) => {
|
||||
if (generation !== this.startGeneration) return;
|
||||
this.emit('fatal-error', {
|
||||
message: error instanceof Error ? error.message : '实时语音启动失败'
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.realtime-app-bridge {
|
||||
width: 0;
|
||||
height: 0;
|
||||
overflow: hidden;
|
||||
}
|
||||
</style>
|
||||
@@ -485,6 +485,18 @@
|
||||
<button class="secondary-button" @click="openPracticeHistory">查看训练记录</button>
|
||||
<button class="primary-button tabbar-safe-action" :disabled="!canStartPractice || busy" @click="start(selectedScenarioId)">再练一次</button>
|
||||
</view>
|
||||
|
||||
<!-- #ifdef APP-PLUS -->
|
||||
<RealtimePracticeAppBridge
|
||||
ref="realtimeAppBridge"
|
||||
@status="handleRealtimeStatus"
|
||||
@trainee-transcript="handleRealtimeTraineeTranscript"
|
||||
@customer-transcript="handleRealtimeCustomerTranscript"
|
||||
@audio-playback-blocked="handleRealtimeAudioPlaybackBlocked"
|
||||
@terminated="handleRealtimeTerminated"
|
||||
@error="handleRealtimeError"
|
||||
/>
|
||||
<!-- #endif -->
|
||||
</view>
|
||||
</template>
|
||||
|
||||
@@ -492,6 +504,7 @@
|
||||
import { computed, nextTick, ref } from 'vue';
|
||||
import { onHide, onShow, onUnload } from '@dcloudio/uni-app';
|
||||
import ChatComposer from '@/components/chat/ChatComposer.vue';
|
||||
import RealtimePracticeAppBridge from '@/components/practice/RealtimePracticeAppBridge.vue';
|
||||
import type {
|
||||
PracticeFinishResponse,
|
||||
PracticePrepCard,
|
||||
@@ -513,7 +526,7 @@ import { searchKnowledge } from '@/services/knowledge';
|
||||
import { resetPageScroll } from '@/services/navigation';
|
||||
import { ensureEmployeePosition } from '@/services/position';
|
||||
import { chooseSpeechAudio, createSpeechPlaybackController, customerVoiceProfile, measureAudioDuration, MENTOR_VOICE_PROFILE, normalizeOssId, transcribeSpeechCapture, transcribeSpeechFile } from '@/services/speech';
|
||||
import { startSpeechCapture } from '@/services/speech-capture';
|
||||
import { isNativeAppRuntime, startSpeechCapture } from '@/services/speech-capture';
|
||||
import type { SpeechCapture, SpeechCaptureSession } from '@/services/speech-capture';
|
||||
import { createPageRequestScope, currentAccountKey, type RequestScopeTicket } from '@/services/request-scope';
|
||||
import { createRealtimePracticeController, createRealtimeSession, fetchRealtimePersonas, invokeRealtimeTool, type RealtimePersonaOption } from '@/services/realtime-practice';
|
||||
@@ -572,6 +585,11 @@ const realtimeAudioBlocked = ref(false);
|
||||
const realtimeSessionId = ref('');
|
||||
const realtimePersonaOptions = ref<RealtimePersonaOption[]>([]);
|
||||
const selectedRealtimePersonaId = ref(typeof storedRealtimePersona === 'string' ? storedRealtimePersona : '');
|
||||
const realtimeAppBridge = ref<{
|
||||
start: (sessionUpdate: Record<string, unknown>, sessionId?: string) => Promise<void>;
|
||||
stop: () => void;
|
||||
resumeAudio: () => Promise<boolean>;
|
||||
} | null>(null);
|
||||
const growthDetailsExpanded = ref(false);
|
||||
const assignments = ref<PracticeAssignment[]>([]);
|
||||
const practiceHistory = ref<PracticeRecord[]>([]);
|
||||
@@ -1010,34 +1028,40 @@ const appendTurn = (role: PracticeRole, text?: string, emotion = 0, synthesizeCu
|
||||
scrollThreadToBottom();
|
||||
};
|
||||
|
||||
const handleRealtimeStatus = (nextStatus: typeof realtimeStatus.value) => {
|
||||
realtimeStatus.value = nextStatus;
|
||||
};
|
||||
const handleRealtimeTraineeTranscript = (text: string) => {
|
||||
if (realtimeActive.value) appendTurn('trainee', text);
|
||||
};
|
||||
const handleRealtimeCustomerTranscript = (text: string) => {
|
||||
if (realtimeActive.value) appendTurn('customer', text, 0, false);
|
||||
};
|
||||
const handleRealtimeAudioPlaybackBlocked = (blocked: boolean) => {
|
||||
if (realtimeActive.value) realtimeAudioBlocked.value = blocked;
|
||||
};
|
||||
const handleRealtimeTerminated = () => {
|
||||
if (!realtimeActive.value) return;
|
||||
realtimeActive.value = false;
|
||||
realtimeStatus.value = 'idle';
|
||||
realtimeAudioBlocked.value = false;
|
||||
realtimeSessionId.value = '';
|
||||
sessionId.value = '';
|
||||
activeAssignmentId.value = undefined;
|
||||
status.value = 'idle';
|
||||
};
|
||||
const handleRealtimeError = (text: string) => {
|
||||
if (realtimeActive.value) message.value = text;
|
||||
};
|
||||
const realtimePractice = createRealtimePracticeController({
|
||||
onStatus: (nextStatus) => {
|
||||
realtimeStatus.value = nextStatus;
|
||||
},
|
||||
onTraineeTranscript: (text) => {
|
||||
if (realtimeActive.value) appendTurn('trainee', text);
|
||||
},
|
||||
onCustomerTranscript: (text) => {
|
||||
if (realtimeActive.value) appendTurn('customer', text, 0, false);
|
||||
},
|
||||
onAudioPlaybackBlocked: (blocked) => {
|
||||
if (realtimeActive.value) realtimeAudioBlocked.value = blocked;
|
||||
},
|
||||
onStatus: handleRealtimeStatus,
|
||||
onTraineeTranscript: handleRealtimeTraineeTranscript,
|
||||
onCustomerTranscript: handleRealtimeCustomerTranscript,
|
||||
onAudioPlaybackBlocked: handleRealtimeAudioPlaybackBlocked,
|
||||
onToolCall: (callId, name, argsJson) =>
|
||||
invokeRealtimeTool(realtimeSessionId.value, callId, name, argsJson).then((res) => res.output),
|
||||
onTerminated: () => {
|
||||
if (!realtimeActive.value) return;
|
||||
realtimeActive.value = false;
|
||||
realtimeStatus.value = 'idle';
|
||||
realtimeAudioBlocked.value = false;
|
||||
realtimeSessionId.value = '';
|
||||
sessionId.value = '';
|
||||
activeAssignmentId.value = undefined;
|
||||
status.value = 'idle';
|
||||
},
|
||||
onError: (text) => {
|
||||
if (realtimeActive.value) message.value = text;
|
||||
}
|
||||
onTerminated: handleRealtimeTerminated,
|
||||
onError: handleRealtimeError
|
||||
});
|
||||
|
||||
const selectRealtimePersona = (item: RealtimePersonaOption) => {
|
||||
@@ -1062,7 +1086,8 @@ const loadRealtimePersonas = async (request: RequestScopeTicket = requests.begin
|
||||
|
||||
const stopRealtime = () => {
|
||||
if (!realtimeActive.value && realtimeStatus.value === 'idle') return;
|
||||
realtimePractice.stop();
|
||||
if (isNativeAppRuntime()) realtimeAppBridge.value?.stop();
|
||||
else realtimePractice.stop();
|
||||
realtimeActive.value = false;
|
||||
realtimeStatus.value = 'idle';
|
||||
realtimeAudioBlocked.value = false;
|
||||
@@ -1093,7 +1118,9 @@ const startRealtime = async () => {
|
||||
if (!realtimeActive.value) return;
|
||||
realtimeSessionId.value = session.sessionId;
|
||||
activeScenarioName.value = session.personaName || '实时陪练';
|
||||
await realtimePractice.start(session.sessionUpdate, session.sessionId);
|
||||
const controller = isNativeAppRuntime() ? realtimeAppBridge.value : realtimePractice;
|
||||
if (!controller) throw new Error('实时语音模块尚未就绪，请稍后重试');
|
||||
await controller.start(session.sessionUpdate, session.sessionId);
|
||||
if (!realtimeActive.value) return;
|
||||
status.value = 'active';
|
||||
message.value = `已连接「${session.personaName}」，请直接开口说话。`;
|
||||
@@ -1107,7 +1134,8 @@ const startRealtime = async () => {
|
||||
};
|
||||
|
||||
const resumeRealtimeAudio = async () => {
|
||||
if (await realtimePractice.resumeAudio()) {
|
||||
const controller = isNativeAppRuntime() ? realtimeAppBridge.value : realtimePractice;
|
||||
if (controller && await controller.resumeAudio()) {
|
||||
message.value = '已恢复实时语音播放。';
|
||||
} else {
|
||||
message.value = '仍无法播放实时语音，请确认设备媒体权限后重试。';
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
export type RealtimeBrowserStatus =
|
||||
| 'idle'
|
||||
| 'connecting'
|
||||
| 'connected'
|
||||
| 'listening'
|
||||
| 'speaking'
|
||||
| 'calling-tool';
|
||||
|
||||
export interface RealtimeBrowserCallbacks {
|
||||
onStatus: (status: RealtimeBrowserStatus) => void;
|
||||
onTraineeTranscript: (text: string) => void;
|
||||
onCustomerTranscript: (text: string) => void;
|
||||
onAudioPlaybackBlocked: (blocked: boolean) => void;
|
||||
onToolCall: (callId: string, name: string, argsJson: string) => Promise<string>;
|
||||
onTerminated: () => void;
|
||||
onError: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface RealtimeBrowserEngine {
|
||||
start: (sessionUpdate: Record<string, unknown>, sessionId?: string) => Promise<void>;
|
||||
stop: () => void;
|
||||
resumeAudio: () => Promise<boolean>;
|
||||
}
|
||||
|
||||
export const isRealtimeBrowserSupported: () => boolean;
|
||||
|
||||
export const createRealtimeBrowserEngine: (
|
||||
callbacks: RealtimeBrowserCallbacks,
|
||||
exchangeSdp: (offerSdp: string, sessionId?: string) => Promise<string>
|
||||
) => RealtimeBrowserEngine;
|
||||
@@ -0,0 +1,302 @@
|
||||
const trimTranscript = (value) => (value || '').trim();
|
||||
const normalizeSdp = (sdp) => `${sdp.replace(/\r\n|\r|\n/g, '\r\n').replace(/\r\n$/, '')}\r\n`;
|
||||
const TOOL_CALL_FALLBACK_OUTPUT = '工具暂时不可用，请直接回应员工';
|
||||
|
||||
export const isRealtimeBrowserSupported = () =>
|
||||
typeof navigator !== 'undefined'
|
||||
&& Boolean(navigator.mediaDevices?.getUserMedia)
|
||||
&& typeof RTCPeerConnection !== 'undefined';
|
||||
|
||||
const waitForIceComplete = (connection) =>
|
||||
new Promise((resolve) => {
|
||||
if (connection.iceGatheringState === 'complete') {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
const previous = connection.onicegatheringstatechange;
|
||||
let timeout;
|
||||
const done = () => {
|
||||
clearTimeout(timeout);
|
||||
connection.onicegatheringstatechange = previous;
|
||||
resolve();
|
||||
};
|
||||
timeout = setTimeout(done, 3000);
|
||||
connection.onicegatheringstatechange = () => {
|
||||
previous?.call(connection, new Event('icegatheringstatechange'));
|
||||
if (connection.iceGatheringState === 'complete') done();
|
||||
};
|
||||
});
|
||||
|
||||
export const createRealtimeBrowserEngine = (callbacks, exchangeSdp) => {
|
||||
let activeAttempt = 0;
|
||||
let connection = null;
|
||||
let microphone = null;
|
||||
let speaker = null;
|
||||
let controlChannel = null;
|
||||
let cancelSessionWait = null;
|
||||
let sessionWaitAttempt = null;
|
||||
let toolCallChain = Promise.resolve();
|
||||
|
||||
const isCurrent = (attempt) => activeAttempt === attempt;
|
||||
|
||||
const releaseResources = () => {
|
||||
controlChannel?.close();
|
||||
controlChannel = null;
|
||||
microphone?.getTracks().forEach((track) => track.stop());
|
||||
microphone = null;
|
||||
if (speaker) {
|
||||
speaker.pause();
|
||||
speaker.srcObject = null;
|
||||
speaker = null;
|
||||
}
|
||||
connection?.close();
|
||||
connection = null;
|
||||
};
|
||||
|
||||
const sendSessionUpdate = (channel, sessionUpdate) => {
|
||||
if (channel.readyState !== 'open') return;
|
||||
channel.send(JSON.stringify({ event_id: `event_${Date.now()}`, ...sessionUpdate }));
|
||||
};
|
||||
|
||||
const handleToolCall = (attempt, event) => {
|
||||
const callId = trimTranscript(event.call_id);
|
||||
const name = trimTranscript(event.name);
|
||||
if (!callId || !name) return;
|
||||
const argsJson = typeof event.arguments === 'string' ? event.arguments : '{}';
|
||||
toolCallChain = toolCallChain.then(async () => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
const channel = controlChannel;
|
||||
if (!channel || channel.readyState !== 'open') return;
|
||||
callbacks.onStatus('calling-tool');
|
||||
let output;
|
||||
try {
|
||||
output = await callbacks.onToolCall(callId, name, argsJson);
|
||||
} catch {
|
||||
output = TOOL_CALL_FALLBACK_OUTPUT;
|
||||
}
|
||||
if (!isCurrent(attempt) || channel.readyState !== 'open') return;
|
||||
channel.send(JSON.stringify({
|
||||
event_id: `event_${Date.now()}`,
|
||||
type: 'conversation.item.create',
|
||||
item: { type: 'function_call_output', call_id: callId, output }
|
||||
}));
|
||||
channel.send(JSON.stringify({ event_id: `event_${Date.now()}`, type: 'response.create' }));
|
||||
callbacks.onStatus('connected');
|
||||
}).catch(() => {
|
||||
// 链内任何意外抛错不得毒化后续工具调用；本次调用软失败，对话继续
|
||||
});
|
||||
};
|
||||
|
||||
const handleEvent = (raw, attempt, provider) => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
let event;
|
||||
try {
|
||||
event = JSON.parse(raw);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (event.type === 'session.created') provider.onSessionCreated();
|
||||
if (event.type === 'session.updated') provider.onSessionUpdated();
|
||||
if (event.type === 'input_audio_buffer.speech_started') callbacks.onStatus('listening');
|
||||
if (event.type === 'response.created') callbacks.onStatus('speaking');
|
||||
if (event.type === 'response.done') callbacks.onStatus('connected');
|
||||
if (event.type === 'conversation.item.input_audio_transcription.completed') {
|
||||
const transcript = trimTranscript(event.transcript);
|
||||
if (transcript) callbacks.onTraineeTranscript(transcript);
|
||||
}
|
||||
if (event.type === 'response.audio_transcript.done') {
|
||||
const transcript = trimTranscript(event.transcript);
|
||||
if (transcript) callbacks.onCustomerTranscript(transcript);
|
||||
}
|
||||
if (event.type === 'response.function_call_arguments.done') {
|
||||
handleToolCall(attempt, event);
|
||||
}
|
||||
if (event.type === 'error') {
|
||||
const error = new Error(event.error?.message || '实时陪练连接异常，请改用录音对练');
|
||||
callbacks.onError(error.message);
|
||||
provider.onSessionError(error);
|
||||
}
|
||||
};
|
||||
|
||||
const bindChannel = (channel, attempt, provider) => {
|
||||
channel.onmessage = (event) => handleEvent(String(event.data || ''), attempt, provider);
|
||||
channel.onerror = () => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
const error = new Error('实时陪练数据通道异常，请改用录音对练');
|
||||
callbacks.onError(error.message);
|
||||
provider.onSessionError(error);
|
||||
};
|
||||
};
|
||||
|
||||
const stop = () => {
|
||||
activeAttempt += 1;
|
||||
cancelSessionWait?.();
|
||||
cancelSessionWait = null;
|
||||
sessionWaitAttempt = null;
|
||||
toolCallChain = Promise.resolve();
|
||||
releaseResources();
|
||||
callbacks.onAudioPlaybackBlocked(false);
|
||||
callbacks.onStatus('idle');
|
||||
};
|
||||
|
||||
const terminate = (attempt, message) => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
callbacks.onError(message);
|
||||
stop();
|
||||
callbacks.onTerminated();
|
||||
};
|
||||
|
||||
const resumeAudio = async () => {
|
||||
if (!speaker) return false;
|
||||
try {
|
||||
await speaker.play();
|
||||
callbacks.onAudioPlaybackBlocked(false);
|
||||
return true;
|
||||
} catch {
|
||||
callbacks.onAudioPlaybackBlocked(true);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
const start = async (sessionUpdate, sessionId) => {
|
||||
if (!isRealtimeBrowserSupported()) throw new Error('当前设备的 WebView 不支持实时语音，请更新系统 WebView 后重试');
|
||||
stop();
|
||||
const attempt = ++activeAttempt;
|
||||
callbacks.onStatus('connecting');
|
||||
|
||||
let localMicrophone = null;
|
||||
let localConnection = null;
|
||||
let localChannel = null;
|
||||
let localSpeaker = null;
|
||||
let readyTimeout;
|
||||
let disconnectTimeout;
|
||||
let sessionUpdateSent = false;
|
||||
let resolveReady;
|
||||
let rejectReady;
|
||||
let sessionReady = null;
|
||||
const releaseLocalResources = () => {
|
||||
if (disconnectTimeout) clearTimeout(disconnectTimeout);
|
||||
localChannel?.close();
|
||||
localMicrophone?.getTracks().forEach((track) => track.stop());
|
||||
if (localSpeaker) {
|
||||
localSpeaker.pause();
|
||||
localSpeaker.srcObject = null;
|
||||
}
|
||||
localConnection?.close();
|
||||
};
|
||||
const ensureCurrent = () => {
|
||||
if (isCurrent(attempt)) return true;
|
||||
releaseLocalResources();
|
||||
return false;
|
||||
};
|
||||
|
||||
try {
|
||||
localMicrophone = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
if (!ensureCurrent()) return;
|
||||
microphone = localMicrophone;
|
||||
localConnection = new RTCPeerConnection({ iceServers: [] });
|
||||
connection = localConnection;
|
||||
const audioSender = localConnection.addTransceiver('audio', { direction: 'sendrecv' }).sender;
|
||||
const provider = {
|
||||
onSessionCreated: () => {
|
||||
if (!isCurrent(attempt) || !localChannel || sessionUpdateSent) return;
|
||||
sessionUpdateSent = true;
|
||||
sendSessionUpdate(localChannel, sessionUpdate);
|
||||
},
|
||||
onSessionUpdated: () => {
|
||||
if (!isCurrent(attempt) || !sessionUpdateSent) return;
|
||||
const track = localMicrophone?.getAudioTracks()[0] || null;
|
||||
void audioSender.replaceTrack(track)
|
||||
.then(() => {
|
||||
if (isCurrent(attempt)) resolveReady?.();
|
||||
})
|
||||
.catch((error) => rejectReady?.(error instanceof Error ? error : new Error('实时陪练麦克风启动失败')));
|
||||
},
|
||||
onSessionError: (error) => {
|
||||
rejectReady?.(error);
|
||||
terminate(attempt, error.message);
|
||||
}
|
||||
};
|
||||
|
||||
localChannel = localConnection.createDataChannel('oai-events');
|
||||
controlChannel = localChannel;
|
||||
bindChannel(localChannel, attempt, provider);
|
||||
localConnection.ondatachannel = (event) => bindChannel(event.channel, attempt, provider);
|
||||
localConnection.ontrack = (event) => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
const stream = event.streams[0] || new MediaStream([event.track]);
|
||||
localSpeaker = new Audio();
|
||||
speaker = localSpeaker;
|
||||
localSpeaker.autoplay = true;
|
||||
localSpeaker.setAttribute('playsinline', 'true');
|
||||
localSpeaker.srcObject = stream;
|
||||
void localSpeaker.play()
|
||||
.then(() => {
|
||||
if (isCurrent(attempt)) callbacks.onAudioPlaybackBlocked(false);
|
||||
})
|
||||
.catch(() => {
|
||||
if (isCurrent(attempt)) callbacks.onAudioPlaybackBlocked(true);
|
||||
});
|
||||
};
|
||||
localConnection.onconnectionstatechange = () => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
if (localConnection?.connectionState === 'failed' || localConnection?.connectionState === 'closed') {
|
||||
terminate(attempt, '实时陪练已断开，请改用录音对练');
|
||||
return;
|
||||
}
|
||||
if (localConnection?.connectionState === 'disconnected') {
|
||||
disconnectTimeout ??= setTimeout(() => {
|
||||
if (localConnection?.connectionState === 'disconnected') {
|
||||
terminate(attempt, '实时陪练已断开，请改用录音对练');
|
||||
}
|
||||
}, 3000);
|
||||
} else if (disconnectTimeout) {
|
||||
clearTimeout(disconnectTimeout);
|
||||
disconnectTimeout = undefined;
|
||||
}
|
||||
};
|
||||
|
||||
const offer = await localConnection.createOffer();
|
||||
if (!ensureCurrent()) return;
|
||||
await localConnection.setLocalDescription(offer);
|
||||
if (!ensureCurrent()) return;
|
||||
await waitForIceComplete(localConnection);
|
||||
if (!ensureCurrent()) return;
|
||||
const offerSdp = localConnection.localDescription?.sdp;
|
||||
if (!offerSdp) throw new Error('实时陪练连接请求创建失败');
|
||||
const answerSdp = await exchangeSdp(normalizeSdp(offerSdp), sessionId);
|
||||
if (!ensureCurrent()) return;
|
||||
sessionReady = new Promise((resolve, reject) => {
|
||||
resolveReady = resolve;
|
||||
rejectReady = reject;
|
||||
readyTimeout = setTimeout(() => reject(new Error('实时陪练会话初始化超时，请使用录音对练')), 15000);
|
||||
});
|
||||
void sessionReady.catch(() => undefined);
|
||||
sessionWaitAttempt = attempt;
|
||||
cancelSessionWait = () => {
|
||||
if (readyTimeout) clearTimeout(readyTimeout);
|
||||
rejectReady?.(new Error('实时陪练已结束'));
|
||||
};
|
||||
await localConnection.setRemoteDescription({ type: 'answer', sdp: normalizeSdp(answerSdp) });
|
||||
if (!ensureCurrent()) return;
|
||||
await sessionReady;
|
||||
if (!ensureCurrent()) return;
|
||||
callbacks.onStatus('connected');
|
||||
} catch (error) {
|
||||
if (!isCurrent(attempt)) {
|
||||
releaseLocalResources();
|
||||
return;
|
||||
}
|
||||
stop();
|
||||
throw error instanceof Error ? error : new Error('实时陪练连接失败，请改用录音对练');
|
||||
} finally {
|
||||
if (readyTimeout) clearTimeout(readyTimeout);
|
||||
if (sessionWaitAttempt === attempt) {
|
||||
sessionWaitAttempt = null;
|
||||
cancelSessionWait = null;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return { start, stop, resumeAudio };
|
||||
};
|
||||
@@ -1,7 +1,11 @@
|
||||
import { apiRequest } from './api';
|
||||
import { isNativeAppRuntime } from './speech-capture';
|
||||
|
||||
type RealtimeStatus = 'idle' | 'connecting' | 'connected' | 'listening' | 'speaking' | 'calling-tool';
|
||||
import { createRealtimeBrowserEngine } from './realtime-browser-engine.js';
|
||||
import type {
|
||||
RealtimeBrowserCallbacks,
|
||||
RealtimeBrowserEngine,
|
||||
RealtimeBrowserStatus
|
||||
} from './realtime-browser-engine.js';
|
||||
|
||||
export interface RealtimePersonaOption {
|
||||
id: string;
|
||||
@@ -28,36 +32,11 @@ interface RealtimeSdpResponse {
|
||||
model: string;
|
||||
}
|
||||
|
||||
interface RealtimeEvent {
|
||||
type?: string;
|
||||
transcript?: string;
|
||||
call_id?: string;
|
||||
name?: string;
|
||||
arguments?: string;
|
||||
error?: { message?: string };
|
||||
export interface RealtimePracticeCallbacks extends RealtimeBrowserCallbacks {
|
||||
onStatus: (status: RealtimeBrowserStatus) => void;
|
||||
}
|
||||
|
||||
interface ProviderEventHandlers {
|
||||
onSessionCreated: () => void;
|
||||
onSessionUpdated: () => void;
|
||||
onSessionError: (error: Error) => void;
|
||||
}
|
||||
|
||||
export interface RealtimePracticeCallbacks {
|
||||
onStatus: (status: RealtimeStatus) => void;
|
||||
onTraineeTranscript: (text: string) => void;
|
||||
onCustomerTranscript: (text: string) => void;
|
||||
onAudioPlaybackBlocked: (blocked: boolean) => void;
|
||||
onToolCall: (callId: string, name: string, argsJson: string) => Promise<string>;
|
||||
onTerminated: () => void;
|
||||
onError: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface RealtimePracticeController {
|
||||
start: (sessionUpdate: Record<string, unknown>, sessionId?: string) => Promise<void>;
|
||||
stop: () => void;
|
||||
resumeAudio: () => Promise<boolean>;
|
||||
}
|
||||
export interface RealtimePracticeController extends RealtimeBrowserEngine {}
|
||||
|
||||
export const fetchRealtimePersonas = () =>
|
||||
apiRequest<RealtimePersonaOption[]>({
|
||||
@@ -82,316 +61,27 @@ export const invokeRealtimeTool = (sessionId: string, callId: string, name: stri
|
||||
timeout: 20000
|
||||
});
|
||||
|
||||
const isSupported = () =>
|
||||
!isNativeAppRuntime()
|
||||
&& typeof navigator !== 'undefined'
|
||||
&& Boolean(navigator.mediaDevices?.getUserMedia)
|
||||
&& typeof RTCPeerConnection !== 'undefined';
|
||||
|
||||
const waitForIceComplete = (connection: RTCPeerConnection) =>
|
||||
new Promise<void>((resolve) => {
|
||||
if (connection.iceGatheringState === 'complete') {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
const previous = connection.onicegatheringstatechange;
|
||||
let timeout: ReturnType<typeof setTimeout>;
|
||||
const done = () => {
|
||||
clearTimeout(timeout);
|
||||
connection.onicegatheringstatechange = previous;
|
||||
resolve();
|
||||
};
|
||||
timeout = setTimeout(done, 3000);
|
||||
connection.onicegatheringstatechange = () => {
|
||||
previous?.call(connection, new Event('icegatheringstatechange'));
|
||||
if (connection.iceGatheringState === 'complete') done();
|
||||
};
|
||||
export const exchangeRealtimeSdp = async (offerSdp: string, sessionId?: string) => {
|
||||
const response = await apiRequest<RealtimeSdpResponse>({
|
||||
url: '/api/train/practice/realtime/sdp',
|
||||
method: 'POST',
|
||||
data: { offerSdp, sessionId: sessionId || null },
|
||||
timeout: 30000
|
||||
});
|
||||
|
||||
const normalizeSdp = (sdp: string) => `${sdp.replace(/\r\n|\r|\n/g, '\r\n').replace(/\r\n$/, '')}\r\n`;
|
||||
const trimTranscript = (value?: string) => (value || '').trim();
|
||||
|
||||
const TOOL_CALL_FALLBACK_OUTPUT = '工具暂时不可用，请直接回应员工';
|
||||
|
||||
export const createRealtimePracticeController = (callbacks: RealtimePracticeCallbacks): RealtimePracticeController => {
|
||||
let activeAttempt = 0;
|
||||
let connection: RTCPeerConnection | null = null;
|
||||
let microphone: MediaStream | null = null;
|
||||
let speaker: HTMLAudioElement | null = null;
|
||||
let controlChannel: RTCDataChannel | null = null;
|
||||
let cancelSessionWait: (() => void) | null = null;
|
||||
let sessionWaitAttempt: number | null = null;
|
||||
let toolCallChain: Promise<void> = Promise.resolve();
|
||||
|
||||
const isCurrent = (attempt: number) => activeAttempt === attempt;
|
||||
|
||||
const releaseResources = () => {
|
||||
controlChannel?.close();
|
||||
controlChannel = null;
|
||||
microphone?.getTracks().forEach((track) => track.stop());
|
||||
microphone = null;
|
||||
if (speaker) {
|
||||
speaker.pause();
|
||||
speaker.srcObject = null;
|
||||
speaker = null;
|
||||
}
|
||||
connection?.close();
|
||||
connection = null;
|
||||
};
|
||||
|
||||
// 服务端下发的 session.update 载荷原样转发，仅补 event_id
|
||||
const sendSessionUpdate = (channel: RTCDataChannel, sessionUpdate: Record<string, unknown>) => {
|
||||
if (channel.readyState !== 'open') return;
|
||||
channel.send(JSON.stringify({ event_id: `event_${Date.now()}`, ...sessionUpdate }));
|
||||
};
|
||||
|
||||
const handleToolCall = (attempt: number, event: RealtimeEvent) => {
|
||||
const callId = trimTranscript(event.call_id);
|
||||
const name = trimTranscript(event.name);
|
||||
if (!callId || !name) return;
|
||||
const argsJson = typeof event.arguments === 'string' ? event.arguments : '{}';
|
||||
// 多次工具调用串行化，避免打断竞态下回传交错
|
||||
toolCallChain = toolCallChain.then(async () => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
const channel = controlChannel;
|
||||
if (!channel || channel.readyState !== 'open') return;
|
||||
callbacks.onStatus('calling-tool');
|
||||
let output: string;
|
||||
try {
|
||||
output = await callbacks.onToolCall(callId, name, argsJson);
|
||||
} catch {
|
||||
output = TOOL_CALL_FALLBACK_OUTPUT;
|
||||
}
|
||||
if (!isCurrent(attempt) || channel.readyState !== 'open') return;
|
||||
channel.send(JSON.stringify({
|
||||
event_id: `event_${Date.now()}`,
|
||||
type: 'conversation.item.create',
|
||||
item: { type: 'function_call_output', call_id: callId, output }
|
||||
}));
|
||||
channel.send(JSON.stringify({ event_id: `event_${Date.now()}`, type: 'response.create' }));
|
||||
callbacks.onStatus('connected');
|
||||
}).catch(() => {
|
||||
// 链内任何意外抛错不得毒化后续工具调用；本次调用软失败，对话继续
|
||||
});
|
||||
};
|
||||
|
||||
const handleEvent = (raw: string, attempt: number, provider: ProviderEventHandlers) => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
let event: RealtimeEvent;
|
||||
try {
|
||||
event = JSON.parse(raw) as RealtimeEvent;
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (event.type === 'session.created') provider.onSessionCreated();
|
||||
if (event.type === 'session.updated') provider.onSessionUpdated();
|
||||
if (event.type === 'input_audio_buffer.speech_started') callbacks.onStatus('listening');
|
||||
if (event.type === 'response.created') callbacks.onStatus('speaking');
|
||||
if (event.type === 'response.done') callbacks.onStatus('connected');
|
||||
if (event.type === 'conversation.item.input_audio_transcription.completed') {
|
||||
const transcript = trimTranscript(event.transcript);
|
||||
if (transcript) callbacks.onTraineeTranscript(transcript);
|
||||
}
|
||||
if (event.type === 'response.audio_transcript.done') {
|
||||
const transcript = trimTranscript(event.transcript);
|
||||
if (transcript) callbacks.onCustomerTranscript(transcript);
|
||||
}
|
||||
if (event.type === 'response.function_call_arguments.done') {
|
||||
handleToolCall(attempt, event);
|
||||
}
|
||||
if (event.type === 'error') {
|
||||
const error = new Error(event.error?.message || '实时陪练连接异常，请改用录音对练');
|
||||
callbacks.onError(error.message);
|
||||
provider.onSessionError(error);
|
||||
}
|
||||
};
|
||||
|
||||
const bindChannel = (channel: RTCDataChannel, attempt: number, provider: ProviderEventHandlers) => {
|
||||
channel.onmessage = (event) => handleEvent(String(event.data || ''), attempt, provider);
|
||||
channel.onerror = () => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
const error = new Error('实时陪练数据通道异常，请改用录音对练');
|
||||
callbacks.onError(error.message);
|
||||
provider.onSessionError(error);
|
||||
};
|
||||
};
|
||||
|
||||
const stop = () => {
|
||||
activeAttempt += 1;
|
||||
cancelSessionWait?.();
|
||||
cancelSessionWait = null;
|
||||
sessionWaitAttempt = null;
|
||||
toolCallChain = Promise.resolve();
|
||||
releaseResources();
|
||||
callbacks.onAudioPlaybackBlocked(false);
|
||||
callbacks.onStatus('idle');
|
||||
};
|
||||
|
||||
const terminate = (attempt: number, message: string) => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
callbacks.onError(message);
|
||||
stop();
|
||||
callbacks.onTerminated();
|
||||
};
|
||||
|
||||
const resumeAudio = async () => {
|
||||
if (!speaker) return false;
|
||||
try {
|
||||
await speaker.play();
|
||||
callbacks.onAudioPlaybackBlocked(false);
|
||||
return true;
|
||||
} catch {
|
||||
callbacks.onAudioPlaybackBlocked(true);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
const start = async (sessionUpdate: Record<string, unknown>, sessionId?: string) => {
|
||||
if (!isSupported()) throw new Error('当前设备不支持实时语音，请使用录音对练');
|
||||
stop();
|
||||
const attempt = ++activeAttempt;
|
||||
callbacks.onStatus('connecting');
|
||||
|
||||
let localMicrophone: MediaStream | null = null;
|
||||
let localConnection: RTCPeerConnection | null = null;
|
||||
let localChannel: RTCDataChannel | null = null;
|
||||
let localSpeaker: HTMLAudioElement | null = null;
|
||||
let readyTimeout: ReturnType<typeof setTimeout> | undefined;
|
||||
let disconnectTimeout: ReturnType<typeof setTimeout> | undefined;
|
||||
let sessionUpdateSent = false;
|
||||
let resolveReady: (() => void) | undefined;
|
||||
let rejectReady: ((error: Error) => void) | undefined;
|
||||
let sessionReady: Promise<void> | null = null;
|
||||
const releaseLocalResources = () => {
|
||||
if (disconnectTimeout) clearTimeout(disconnectTimeout);
|
||||
localChannel?.close();
|
||||
localMicrophone?.getTracks().forEach((track) => track.stop());
|
||||
if (localSpeaker) {
|
||||
localSpeaker.pause();
|
||||
localSpeaker.srcObject = null;
|
||||
return response.answerSdp;
|
||||
};
|
||||
|
||||
export const createRealtimePracticeController = (
|
||||
callbacks: RealtimePracticeCallbacks
|
||||
): RealtimePracticeController => {
|
||||
const engine = createRealtimeBrowserEngine(callbacks, exchangeRealtimeSdp);
|
||||
return {
|
||||
...engine,
|
||||
start: async (sessionUpdate, sessionId) => {
|
||||
if (isNativeAppRuntime()) {
|
||||
throw new Error('APP 实时语音应由 WebView 桥接启动');
|
||||
}
|
||||
localConnection?.close();
|
||||
};
|
||||
const ensureCurrent = () => {
|
||||
if (isCurrent(attempt)) return true;
|
||||
releaseLocalResources();
|
||||
return false;
|
||||
};
|
||||
|
||||
try {
|
||||
localMicrophone = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
if (!ensureCurrent()) return;
|
||||
microphone = localMicrophone;
|
||||
localConnection = new RTCPeerConnection({ iceServers: [] });
|
||||
connection = localConnection;
|
||||
const audioSender = localConnection.addTransceiver('audio', { direction: 'sendrecv' }).sender;
|
||||
const provider: ProviderEventHandlers = {
|
||||
onSessionCreated: () => {
|
||||
if (!isCurrent(attempt) || !localChannel || sessionUpdateSent) return;
|
||||
sessionUpdateSent = true;
|
||||
sendSessionUpdate(localChannel, sessionUpdate);
|
||||
},
|
||||
onSessionUpdated: () => {
|
||||
if (!isCurrent(attempt) || !sessionUpdateSent) return;
|
||||
const track = localMicrophone?.getAudioTracks()[0] || null;
|
||||
void audioSender.replaceTrack(track)
|
||||
.then(() => {
|
||||
if (isCurrent(attempt)) resolveReady?.();
|
||||
})
|
||||
.catch((error: unknown) => rejectReady?.(error instanceof Error ? error : new Error('实时陪练麦克风启动失败')));
|
||||
},
|
||||
onSessionError: (error) => {
|
||||
rejectReady?.(error);
|
||||
terminate(attempt, error.message);
|
||||
}
|
||||
};
|
||||
|
||||
localChannel = localConnection.createDataChannel('oai-events');
|
||||
controlChannel = localChannel;
|
||||
bindChannel(localChannel, attempt, provider);
|
||||
localConnection.ondatachannel = (event) => bindChannel(event.channel, attempt, provider);
|
||||
localConnection.ontrack = (event) => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
const stream = event.streams[0] || new MediaStream([event.track]);
|
||||
localSpeaker = new Audio();
|
||||
speaker = localSpeaker;
|
||||
localSpeaker.autoplay = true;
|
||||
localSpeaker.setAttribute('playsinline', 'true');
|
||||
localSpeaker.srcObject = stream;
|
||||
void localSpeaker.play()
|
||||
.then(() => {
|
||||
if (isCurrent(attempt)) callbacks.onAudioPlaybackBlocked(false);
|
||||
})
|
||||
.catch(() => {
|
||||
if (isCurrent(attempt)) callbacks.onAudioPlaybackBlocked(true);
|
||||
});
|
||||
};
|
||||
localConnection.onconnectionstatechange = () => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
if (localConnection?.connectionState === 'failed' || localConnection?.connectionState === 'closed') {
|
||||
terminate(attempt, '实时陪练已断开，请改用录音对练');
|
||||
return;
|
||||
}
|
||||
if (localConnection?.connectionState === 'disconnected') {
|
||||
disconnectTimeout ??= setTimeout(() => {
|
||||
if (localConnection?.connectionState === 'disconnected') {
|
||||
terminate(attempt, '实时陪练已断开，请改用录音对练');
|
||||
}
|
||||
}, 3000);
|
||||
} else if (disconnectTimeout) {
|
||||
clearTimeout(disconnectTimeout);
|
||||
disconnectTimeout = undefined;
|
||||
}
|
||||
};
|
||||
|
||||
const offer = await localConnection.createOffer();
|
||||
if (!ensureCurrent()) return;
|
||||
await localConnection.setLocalDescription(offer);
|
||||
if (!ensureCurrent()) return;
|
||||
await waitForIceComplete(localConnection);
|
||||
if (!ensureCurrent()) return;
|
||||
const offerSdp = localConnection.localDescription?.sdp;
|
||||
if (!offerSdp) throw new Error('实时陪练连接请求创建失败');
|
||||
const response = await apiRequest<RealtimeSdpResponse>({
|
||||
url: '/api/train/practice/realtime/sdp',
|
||||
method: 'POST',
|
||||
data: { offerSdp: normalizeSdp(offerSdp), sessionId: sessionId || null },
|
||||
timeout: 30000
|
||||
});
|
||||
if (!ensureCurrent()) return;
|
||||
sessionReady = new Promise<void>((resolve, reject) => {
|
||||
resolveReady = resolve;
|
||||
rejectReady = reject;
|
||||
readyTimeout = setTimeout(() => reject(new Error('实时陪练会话初始化超时，请使用录音对练')), 15000);
|
||||
});
|
||||
void sessionReady.catch(() => undefined);
|
||||
sessionWaitAttempt = attempt;
|
||||
cancelSessionWait = () => {
|
||||
if (readyTimeout) clearTimeout(readyTimeout);
|
||||
rejectReady?.(new Error('实时陪练已结束'));
|
||||
};
|
||||
await localConnection.setRemoteDescription({ type: 'answer', sdp: normalizeSdp(response.answerSdp) });
|
||||
if (!ensureCurrent()) return;
|
||||
const waitForSessionReady = sessionReady;
|
||||
if (!waitForSessionReady) throw new Error('实时陪练会话初始化失败');
|
||||
await waitForSessionReady;
|
||||
if (!ensureCurrent()) return;
|
||||
callbacks.onStatus('connected');
|
||||
} catch (error) {
|
||||
if (!isCurrent(attempt)) {
|
||||
releaseLocalResources();
|
||||
return;
|
||||
}
|
||||
stop();
|
||||
throw error instanceof Error ? error : new Error('实时陪练连接失败，请改用录音对练');
|
||||
} finally {
|
||||
if (readyTimeout) clearTimeout(readyTimeout);
|
||||
if (sessionWaitAttempt === attempt) {
|
||||
sessionWaitAttempt = null;
|
||||
cancelSessionWait = null;
|
||||
}
|
||||
await engine.start(sessionUpdate, sessionId);
|
||||
}
|
||||
};
|
||||
|
||||
return { start, stop, resumeAudio };
|
||||
};
|
||||
|
||||
@@ -5,39 +5,52 @@ import test from 'node:test';
|
||||
const source = (path) => readFile(new URL(path, import.meta.url), 'utf8');
|
||||
|
||||
test('实时陪练通过受认证代理接入，前端不携带供应商密钥、不自组会话配置', async () => {
|
||||
const [service, page, backendService, backendSessionService, backendPersonaRegistry] = await Promise.all([
|
||||
const [service, engine, appBridge, page, backendService, backendSessionService, backendPersonaRegistry] = await Promise.all([
|
||||
source('../src/services/realtime-practice.ts'),
|
||||
source('../src/services/realtime-browser-engine.js'),
|
||||
source('../src/components/practice/RealtimePracticeAppBridge.vue'),
|
||||
source('../src/pages/user/practice/index.vue'),
|
||||
source('../../backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/service/AihrRealtimePracticeService.java'),
|
||||
source('../../backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/service/AihrRealtimeSessionService.java'),
|
||||
source('../../backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/service/AihrRealtimePersonaRegistry.java')
|
||||
]);
|
||||
const transport = `${service}\n${engine}`;
|
||||
|
||||
// 传输与事件协议
|
||||
assert.match(service, /\/api\/train\/practice\/realtime\/sdp/);
|
||||
assert.match(service, /conversation\.item\.input_audio_transcription\.completed/);
|
||||
assert.match(service, /response\.audio_transcript\.done/);
|
||||
assert.match(service, /session\.created/);
|
||||
assert.match(service, /session\.updated/);
|
||||
assert.match(service, /activeAttempt/);
|
||||
assert.match(service, /resumeAudio/);
|
||||
assert.match(transport, /conversation\.item\.input_audio_transcription\.completed/);
|
||||
assert.match(transport, /response\.audio_transcript\.done/);
|
||||
assert.match(transport, /session\.created/);
|
||||
assert.match(transport, /session\.updated/);
|
||||
assert.match(transport, /activeAttempt/);
|
||||
assert.match(transport, /resumeAudio/);
|
||||
|
||||
// 方案B 架构：session.update 由服务端组装，浏览器只原样转发；instructions/音色不下放前端
|
||||
assert.match(service, /\/api\/train\/practice\/realtime\/session/);
|
||||
assert.match(service, /sendSessionUpdate/);
|
||||
assert.doesNotMatch(service, /instructions/);
|
||||
assert.doesNotMatch(service, /REALTIME_PRACTICE_VOICES|createRealtimePracticeSessionUpdate|normalizeRealtimePracticeVoice/);
|
||||
assert.doesNotMatch(service, /semantic_vad/);
|
||||
assert.doesNotMatch(service, /DASHSCOPE_API_KEY|Authorization:\s*Bearer/);
|
||||
assert.match(transport, /sendSessionUpdate/);
|
||||
assert.doesNotMatch(transport, /instructions/);
|
||||
assert.doesNotMatch(transport, /REALTIME_PRACTICE_VOICES|createRealtimePracticeSessionUpdate|normalizeRealtimePracticeVoice/);
|
||||
assert.doesNotMatch(transport, /semantic_vad/);
|
||||
assert.doesNotMatch(transport, /DASHSCOPE_API_KEY|Authorization:\s*Bearer/);
|
||||
|
||||
// function calling 工具桥：事件分支、串行链、软着陆兜底、回传协议
|
||||
assert.match(service, /\/api\/train\/practice\/realtime\/tools\/invoke/);
|
||||
assert.match(service, /response\.function_call_arguments\.done/);
|
||||
assert.match(service, /function_call_output/);
|
||||
assert.match(service, /response\.create/);
|
||||
assert.match(service, /toolCallChain/);
|
||||
assert.match(service, /工具暂时不可用，请直接回应员工/);
|
||||
assert.match(service, /calling-tool/);
|
||||
assert.match(transport, /response\.function_call_arguments\.done/);
|
||||
assert.match(transport, /function_call_output/);
|
||||
assert.match(transport, /response\.create/);
|
||||
assert.match(transport, /toolCallChain/);
|
||||
assert.match(transport, /工具暂时不可用，请直接回应员工/);
|
||||
assert.match(transport, /calling-tool/);
|
||||
|
||||
// App-Plus：WebRTC 在 renderjs/WebView 执行，鉴权 SDP 与工具调用回到逻辑层。
|
||||
assert.match(appBridge, /lang="renderjs"/);
|
||||
assert.match(appBridge, /createRealtimeBrowserEngine/);
|
||||
assert.match(appBridge, /exchangeRealtimeSdp/);
|
||||
assert.match(appBridge, /invokeRealtimeTool/);
|
||||
assert.match(appBridge, /owner.*callMethod\('onBridgeEvent'/);
|
||||
assert.match(page, /RealtimePracticeAppBridge/);
|
||||
assert.match(page, /isNativeAppRuntime\(\) \? realtimeAppBridge\.value : realtimePractice/);
|
||||
assert.doesNotMatch(transport, /!isNativeAppRuntime\(\)/);
|
||||
|
||||
// 服务端是会话配置的唯一组装点
|
||||
assert.match(backendSessionService, /session\.update/);
|
||||
@@ -65,7 +78,7 @@ test('实时陪练通过受认证代理接入，前端不携带供应商密钥
|
||||
});
|
||||
|
||||
test('工具调用串行化与竞态防护：链式执行、attempt 双检、stop 重置', async () => {
|
||||
const service = await source('../src/services/realtime-practice.ts');
|
||||
const service = await source('../src/services/realtime-browser-engine.js');
|
||||
|
||||
// 串行链必须基于上一次调用收尾，且带末端 catch 防毒化
|
||||
assert.match(service, /toolCallChain = toolCallChain\.then/);
|
||||
|
||||
Reference in New Issue
Block a user