feat(practice): run realtime practice on App-Plus through renderjs
App-Plus has no WebRTC or getUserMedia in the logic layer, so realtime practice was H5-only and the native entry had to advertise a record-then- transcribe downgrade. Move the media/WebRTC engine into `realtime-browser-engine.js` and drive it from `RealtimePracticeAppBridge.vue`, whose renderjs script runs inside the system WebView where those APIs do exist. The logic layer keeps the authenticated calls: SDP exchange and `search_knowledge` tool invocation stay server-proxied, so the access token is never handed to the view layer. Both sides talk over the renderjs bridge only in session payloads the server already assembled. Verified against a real App build (`app-renderjs.js` holds RTCPeerConnection and getUserMedia; `app-service.js` holds the SDP endpoint, renderjs side has zero hits). Microphone permission, live transcription, remote playback, foreground/background switching and disconnect fallback still require Android and iOS device verification. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,30 @@
|
||||
export type RealtimeBrowserStatus =
|
||||
| 'idle'
|
||||
| 'connecting'
|
||||
| 'connected'
|
||||
| 'listening'
|
||||
| 'speaking'
|
||||
| 'calling-tool';
|
||||
|
||||
export interface RealtimeBrowserCallbacks {
|
||||
onStatus: (status: RealtimeBrowserStatus) => void;
|
||||
onTraineeTranscript: (text: string) => void;
|
||||
onCustomerTranscript: (text: string) => void;
|
||||
onAudioPlaybackBlocked: (blocked: boolean) => void;
|
||||
onToolCall: (callId: string, name: string, argsJson: string) => Promise<string>;
|
||||
onTerminated: () => void;
|
||||
onError: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface RealtimeBrowserEngine {
|
||||
start: (sessionUpdate: Record<string, unknown>, sessionId?: string) => Promise<void>;
|
||||
stop: () => void;
|
||||
resumeAudio: () => Promise<boolean>;
|
||||
}
|
||||
|
||||
export const isRealtimeBrowserSupported: () => boolean;
|
||||
|
||||
export const createRealtimeBrowserEngine: (
|
||||
callbacks: RealtimeBrowserCallbacks,
|
||||
exchangeSdp: (offerSdp: string, sessionId?: string) => Promise<string>
|
||||
) => RealtimeBrowserEngine;
|
||||
@@ -0,0 +1,302 @@
|
||||
const trimTranscript = (value) => (value || '').trim();
|
||||
const normalizeSdp = (sdp) => `${sdp.replace(/\r\n|\r|\n/g, '\r\n').replace(/\r\n$/, '')}\r\n`;
|
||||
const TOOL_CALL_FALLBACK_OUTPUT = '工具暂时不可用，请直接回应员工';
|
||||
|
||||
export const isRealtimeBrowserSupported = () =>
|
||||
typeof navigator !== 'undefined'
|
||||
&& Boolean(navigator.mediaDevices?.getUserMedia)
|
||||
&& typeof RTCPeerConnection !== 'undefined';
|
||||
|
||||
const waitForIceComplete = (connection) =>
|
||||
new Promise((resolve) => {
|
||||
if (connection.iceGatheringState === 'complete') {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
const previous = connection.onicegatheringstatechange;
|
||||
let timeout;
|
||||
const done = () => {
|
||||
clearTimeout(timeout);
|
||||
connection.onicegatheringstatechange = previous;
|
||||
resolve();
|
||||
};
|
||||
timeout = setTimeout(done, 3000);
|
||||
connection.onicegatheringstatechange = () => {
|
||||
previous?.call(connection, new Event('icegatheringstatechange'));
|
||||
if (connection.iceGatheringState === 'complete') done();
|
||||
};
|
||||
});
|
||||
|
||||
export const createRealtimeBrowserEngine = (callbacks, exchangeSdp) => {
|
||||
let activeAttempt = 0;
|
||||
let connection = null;
|
||||
let microphone = null;
|
||||
let speaker = null;
|
||||
let controlChannel = null;
|
||||
let cancelSessionWait = null;
|
||||
let sessionWaitAttempt = null;
|
||||
let toolCallChain = Promise.resolve();
|
||||
|
||||
const isCurrent = (attempt) => activeAttempt === attempt;
|
||||
|
||||
const releaseResources = () => {
|
||||
controlChannel?.close();
|
||||
controlChannel = null;
|
||||
microphone?.getTracks().forEach((track) => track.stop());
|
||||
microphone = null;
|
||||
if (speaker) {
|
||||
speaker.pause();
|
||||
speaker.srcObject = null;
|
||||
speaker = null;
|
||||
}
|
||||
connection?.close();
|
||||
connection = null;
|
||||
};
|
||||
|
||||
const sendSessionUpdate = (channel, sessionUpdate) => {
|
||||
if (channel.readyState !== 'open') return;
|
||||
channel.send(JSON.stringify({ event_id: `event_${Date.now()}`, ...sessionUpdate }));
|
||||
};
|
||||
|
||||
const handleToolCall = (attempt, event) => {
|
||||
const callId = trimTranscript(event.call_id);
|
||||
const name = trimTranscript(event.name);
|
||||
if (!callId || !name) return;
|
||||
const argsJson = typeof event.arguments === 'string' ? event.arguments : '{}';
|
||||
toolCallChain = toolCallChain.then(async () => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
const channel = controlChannel;
|
||||
if (!channel || channel.readyState !== 'open') return;
|
||||
callbacks.onStatus('calling-tool');
|
||||
let output;
|
||||
try {
|
||||
output = await callbacks.onToolCall(callId, name, argsJson);
|
||||
} catch {
|
||||
output = TOOL_CALL_FALLBACK_OUTPUT;
|
||||
}
|
||||
if (!isCurrent(attempt) || channel.readyState !== 'open') return;
|
||||
channel.send(JSON.stringify({
|
||||
event_id: `event_${Date.now()}`,
|
||||
type: 'conversation.item.create',
|
||||
item: { type: 'function_call_output', call_id: callId, output }
|
||||
}));
|
||||
channel.send(JSON.stringify({ event_id: `event_${Date.now()}`, type: 'response.create' }));
|
||||
callbacks.onStatus('connected');
|
||||
}).catch(() => {
|
||||
// 链内任何意外抛错不得毒化后续工具调用；本次调用软失败，对话继续
|
||||
});
|
||||
};
|
||||
|
||||
const handleEvent = (raw, attempt, provider) => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
let event;
|
||||
try {
|
||||
event = JSON.parse(raw);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (event.type === 'session.created') provider.onSessionCreated();
|
||||
if (event.type === 'session.updated') provider.onSessionUpdated();
|
||||
if (event.type === 'input_audio_buffer.speech_started') callbacks.onStatus('listening');
|
||||
if (event.type === 'response.created') callbacks.onStatus('speaking');
|
||||
if (event.type === 'response.done') callbacks.onStatus('connected');
|
||||
if (event.type === 'conversation.item.input_audio_transcription.completed') {
|
||||
const transcript = trimTranscript(event.transcript);
|
||||
if (transcript) callbacks.onTraineeTranscript(transcript);
|
||||
}
|
||||
if (event.type === 'response.audio_transcript.done') {
|
||||
const transcript = trimTranscript(event.transcript);
|
||||
if (transcript) callbacks.onCustomerTranscript(transcript);
|
||||
}
|
||||
if (event.type === 'response.function_call_arguments.done') {
|
||||
handleToolCall(attempt, event);
|
||||
}
|
||||
if (event.type === 'error') {
|
||||
const error = new Error(event.error?.message || '实时陪练连接异常，请改用录音对练');
|
||||
callbacks.onError(error.message);
|
||||
provider.onSessionError(error);
|
||||
}
|
||||
};
|
||||
|
||||
const bindChannel = (channel, attempt, provider) => {
|
||||
channel.onmessage = (event) => handleEvent(String(event.data || ''), attempt, provider);
|
||||
channel.onerror = () => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
const error = new Error('实时陪练数据通道异常，请改用录音对练');
|
||||
callbacks.onError(error.message);
|
||||
provider.onSessionError(error);
|
||||
};
|
||||
};
|
||||
|
||||
const stop = () => {
|
||||
activeAttempt += 1;
|
||||
cancelSessionWait?.();
|
||||
cancelSessionWait = null;
|
||||
sessionWaitAttempt = null;
|
||||
toolCallChain = Promise.resolve();
|
||||
releaseResources();
|
||||
callbacks.onAudioPlaybackBlocked(false);
|
||||
callbacks.onStatus('idle');
|
||||
};
|
||||
|
||||
const terminate = (attempt, message) => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
callbacks.onError(message);
|
||||
stop();
|
||||
callbacks.onTerminated();
|
||||
};
|
||||
|
||||
const resumeAudio = async () => {
|
||||
if (!speaker) return false;
|
||||
try {
|
||||
await speaker.play();
|
||||
callbacks.onAudioPlaybackBlocked(false);
|
||||
return true;
|
||||
} catch {
|
||||
callbacks.onAudioPlaybackBlocked(true);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
const start = async (sessionUpdate, sessionId) => {
|
||||
if (!isRealtimeBrowserSupported()) throw new Error('当前设备的 WebView 不支持实时语音，请更新系统 WebView 后重试');
|
||||
stop();
|
||||
const attempt = ++activeAttempt;
|
||||
callbacks.onStatus('connecting');
|
||||
|
||||
let localMicrophone = null;
|
||||
let localConnection = null;
|
||||
let localChannel = null;
|
||||
let localSpeaker = null;
|
||||
let readyTimeout;
|
||||
let disconnectTimeout;
|
||||
let sessionUpdateSent = false;
|
||||
let resolveReady;
|
||||
let rejectReady;
|
||||
let sessionReady = null;
|
||||
const releaseLocalResources = () => {
|
||||
if (disconnectTimeout) clearTimeout(disconnectTimeout);
|
||||
localChannel?.close();
|
||||
localMicrophone?.getTracks().forEach((track) => track.stop());
|
||||
if (localSpeaker) {
|
||||
localSpeaker.pause();
|
||||
localSpeaker.srcObject = null;
|
||||
}
|
||||
localConnection?.close();
|
||||
};
|
||||
const ensureCurrent = () => {
|
||||
if (isCurrent(attempt)) return true;
|
||||
releaseLocalResources();
|
||||
return false;
|
||||
};
|
||||
|
||||
try {
|
||||
localMicrophone = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
if (!ensureCurrent()) return;
|
||||
microphone = localMicrophone;
|
||||
localConnection = new RTCPeerConnection({ iceServers: [] });
|
||||
connection = localConnection;
|
||||
const audioSender = localConnection.addTransceiver('audio', { direction: 'sendrecv' }).sender;
|
||||
const provider = {
|
||||
onSessionCreated: () => {
|
||||
if (!isCurrent(attempt) || !localChannel || sessionUpdateSent) return;
|
||||
sessionUpdateSent = true;
|
||||
sendSessionUpdate(localChannel, sessionUpdate);
|
||||
},
|
||||
onSessionUpdated: () => {
|
||||
if (!isCurrent(attempt) || !sessionUpdateSent) return;
|
||||
const track = localMicrophone?.getAudioTracks()[0] || null;
|
||||
void audioSender.replaceTrack(track)
|
||||
.then(() => {
|
||||
if (isCurrent(attempt)) resolveReady?.();
|
||||
})
|
||||
.catch((error) => rejectReady?.(error instanceof Error ? error : new Error('实时陪练麦克风启动失败')));
|
||||
},
|
||||
onSessionError: (error) => {
|
||||
rejectReady?.(error);
|
||||
terminate(attempt, error.message);
|
||||
}
|
||||
};
|
||||
|
||||
localChannel = localConnection.createDataChannel('oai-events');
|
||||
controlChannel = localChannel;
|
||||
bindChannel(localChannel, attempt, provider);
|
||||
localConnection.ondatachannel = (event) => bindChannel(event.channel, attempt, provider);
|
||||
localConnection.ontrack = (event) => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
const stream = event.streams[0] || new MediaStream([event.track]);
|
||||
localSpeaker = new Audio();
|
||||
speaker = localSpeaker;
|
||||
localSpeaker.autoplay = true;
|
||||
localSpeaker.setAttribute('playsinline', 'true');
|
||||
localSpeaker.srcObject = stream;
|
||||
void localSpeaker.play()
|
||||
.then(() => {
|
||||
if (isCurrent(attempt)) callbacks.onAudioPlaybackBlocked(false);
|
||||
})
|
||||
.catch(() => {
|
||||
if (isCurrent(attempt)) callbacks.onAudioPlaybackBlocked(true);
|
||||
});
|
||||
};
|
||||
localConnection.onconnectionstatechange = () => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
if (localConnection?.connectionState === 'failed' || localConnection?.connectionState === 'closed') {
|
||||
terminate(attempt, '实时陪练已断开，请改用录音对练');
|
||||
return;
|
||||
}
|
||||
if (localConnection?.connectionState === 'disconnected') {
|
||||
disconnectTimeout ??= setTimeout(() => {
|
||||
if (localConnection?.connectionState === 'disconnected') {
|
||||
terminate(attempt, '实时陪练已断开，请改用录音对练');
|
||||
}
|
||||
}, 3000);
|
||||
} else if (disconnectTimeout) {
|
||||
clearTimeout(disconnectTimeout);
|
||||
disconnectTimeout = undefined;
|
||||
}
|
||||
};
|
||||
|
||||
const offer = await localConnection.createOffer();
|
||||
if (!ensureCurrent()) return;
|
||||
await localConnection.setLocalDescription(offer);
|
||||
if (!ensureCurrent()) return;
|
||||
await waitForIceComplete(localConnection);
|
||||
if (!ensureCurrent()) return;
|
||||
const offerSdp = localConnection.localDescription?.sdp;
|
||||
if (!offerSdp) throw new Error('实时陪练连接请求创建失败');
|
||||
const answerSdp = await exchangeSdp(normalizeSdp(offerSdp), sessionId);
|
||||
if (!ensureCurrent()) return;
|
||||
sessionReady = new Promise((resolve, reject) => {
|
||||
resolveReady = resolve;
|
||||
rejectReady = reject;
|
||||
readyTimeout = setTimeout(() => reject(new Error('实时陪练会话初始化超时，请使用录音对练')), 15000);
|
||||
});
|
||||
void sessionReady.catch(() => undefined);
|
||||
sessionWaitAttempt = attempt;
|
||||
cancelSessionWait = () => {
|
||||
if (readyTimeout) clearTimeout(readyTimeout);
|
||||
rejectReady?.(new Error('实时陪练已结束'));
|
||||
};
|
||||
await localConnection.setRemoteDescription({ type: 'answer', sdp: normalizeSdp(answerSdp) });
|
||||
if (!ensureCurrent()) return;
|
||||
await sessionReady;
|
||||
if (!ensureCurrent()) return;
|
||||
callbacks.onStatus('connected');
|
||||
} catch (error) {
|
||||
if (!isCurrent(attempt)) {
|
||||
releaseLocalResources();
|
||||
return;
|
||||
}
|
||||
stop();
|
||||
throw error instanceof Error ? error : new Error('实时陪练连接失败，请改用录音对练');
|
||||
} finally {
|
||||
if (readyTimeout) clearTimeout(readyTimeout);
|
||||
if (sessionWaitAttempt === attempt) {
|
||||
sessionWaitAttempt = null;
|
||||
cancelSessionWait = null;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return { start, stop, resumeAudio };
|
||||
};
|
||||
@@ -1,7 +1,11 @@
|
||||
import { apiRequest } from './api';
|
||||
import { isNativeAppRuntime } from './speech-capture';
|
||||
|
||||
type RealtimeStatus = 'idle' | 'connecting' | 'connected' | 'listening' | 'speaking' | 'calling-tool';
|
||||
import { createRealtimeBrowserEngine } from './realtime-browser-engine.js';
|
||||
import type {
|
||||
RealtimeBrowserCallbacks,
|
||||
RealtimeBrowserEngine,
|
||||
RealtimeBrowserStatus
|
||||
} from './realtime-browser-engine.js';
|
||||
|
||||
export interface RealtimePersonaOption {
|
||||
id: string;
|
||||
@@ -28,36 +32,11 @@ interface RealtimeSdpResponse {
|
||||
model: string;
|
||||
}
|
||||
|
||||
interface RealtimeEvent {
|
||||
type?: string;
|
||||
transcript?: string;
|
||||
call_id?: string;
|
||||
name?: string;
|
||||
arguments?: string;
|
||||
error?: { message?: string };
|
||||
export interface RealtimePracticeCallbacks extends RealtimeBrowserCallbacks {
|
||||
onStatus: (status: RealtimeBrowserStatus) => void;
|
||||
}
|
||||
|
||||
interface ProviderEventHandlers {
|
||||
onSessionCreated: () => void;
|
||||
onSessionUpdated: () => void;
|
||||
onSessionError: (error: Error) => void;
|
||||
}
|
||||
|
||||
export interface RealtimePracticeCallbacks {
|
||||
onStatus: (status: RealtimeStatus) => void;
|
||||
onTraineeTranscript: (text: string) => void;
|
||||
onCustomerTranscript: (text: string) => void;
|
||||
onAudioPlaybackBlocked: (blocked: boolean) => void;
|
||||
onToolCall: (callId: string, name: string, argsJson: string) => Promise<string>;
|
||||
onTerminated: () => void;
|
||||
onError: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface RealtimePracticeController {
|
||||
start: (sessionUpdate: Record<string, unknown>, sessionId?: string) => Promise<void>;
|
||||
stop: () => void;
|
||||
resumeAudio: () => Promise<boolean>;
|
||||
}
|
||||
export interface RealtimePracticeController extends RealtimeBrowserEngine {}
|
||||
|
||||
export const fetchRealtimePersonas = () =>
|
||||
apiRequest<RealtimePersonaOption[]>({
|
||||
@@ -82,316 +61,27 @@ export const invokeRealtimeTool = (sessionId: string, callId: string, name: stri
|
||||
timeout: 20000
|
||||
});
|
||||
|
||||
const isSupported = () =>
|
||||
!isNativeAppRuntime()
|
||||
&& typeof navigator !== 'undefined'
|
||||
&& Boolean(navigator.mediaDevices?.getUserMedia)
|
||||
&& typeof RTCPeerConnection !== 'undefined';
|
||||
|
||||
const waitForIceComplete = (connection: RTCPeerConnection) =>
|
||||
new Promise<void>((resolve) => {
|
||||
if (connection.iceGatheringState === 'complete') {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
const previous = connection.onicegatheringstatechange;
|
||||
let timeout: ReturnType<typeof setTimeout>;
|
||||
const done = () => {
|
||||
clearTimeout(timeout);
|
||||
connection.onicegatheringstatechange = previous;
|
||||
resolve();
|
||||
};
|
||||
timeout = setTimeout(done, 3000);
|
||||
connection.onicegatheringstatechange = () => {
|
||||
previous?.call(connection, new Event('icegatheringstatechange'));
|
||||
if (connection.iceGatheringState === 'complete') done();
|
||||
};
|
||||
export const exchangeRealtimeSdp = async (offerSdp: string, sessionId?: string) => {
|
||||
const response = await apiRequest<RealtimeSdpResponse>({
|
||||
url: '/api/train/practice/realtime/sdp',
|
||||
method: 'POST',
|
||||
data: { offerSdp, sessionId: sessionId || null },
|
||||
timeout: 30000
|
||||
});
|
||||
|
||||
const normalizeSdp = (sdp: string) => `${sdp.replace(/\r\n|\r|\n/g, '\r\n').replace(/\r\n$/, '')}\r\n`;
|
||||
const trimTranscript = (value?: string) => (value || '').trim();
|
||||
|
||||
const TOOL_CALL_FALLBACK_OUTPUT = '工具暂时不可用，请直接回应员工';
|
||||
|
||||
export const createRealtimePracticeController = (callbacks: RealtimePracticeCallbacks): RealtimePracticeController => {
|
||||
let activeAttempt = 0;
|
||||
let connection: RTCPeerConnection | null = null;
|
||||
let microphone: MediaStream | null = null;
|
||||
let speaker: HTMLAudioElement | null = null;
|
||||
let controlChannel: RTCDataChannel | null = null;
|
||||
let cancelSessionWait: (() => void) | null = null;
|
||||
let sessionWaitAttempt: number | null = null;
|
||||
let toolCallChain: Promise<void> = Promise.resolve();
|
||||
|
||||
const isCurrent = (attempt: number) => activeAttempt === attempt;
|
||||
|
||||
const releaseResources = () => {
|
||||
controlChannel?.close();
|
||||
controlChannel = null;
|
||||
microphone?.getTracks().forEach((track) => track.stop());
|
||||
microphone = null;
|
||||
if (speaker) {
|
||||
speaker.pause();
|
||||
speaker.srcObject = null;
|
||||
speaker = null;
|
||||
}
|
||||
connection?.close();
|
||||
connection = null;
|
||||
};
|
||||
|
||||
// 服务端下发的 session.update 载荷原样转发，仅补 event_id
|
||||
const sendSessionUpdate = (channel: RTCDataChannel, sessionUpdate: Record<string, unknown>) => {
|
||||
if (channel.readyState !== 'open') return;
|
||||
channel.send(JSON.stringify({ event_id: `event_${Date.now()}`, ...sessionUpdate }));
|
||||
};
|
||||
|
||||
const handleToolCall = (attempt: number, event: RealtimeEvent) => {
|
||||
const callId = trimTranscript(event.call_id);
|
||||
const name = trimTranscript(event.name);
|
||||
if (!callId || !name) return;
|
||||
const argsJson = typeof event.arguments === 'string' ? event.arguments : '{}';
|
||||
// 多次工具调用串行化，避免打断竞态下回传交错
|
||||
toolCallChain = toolCallChain.then(async () => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
const channel = controlChannel;
|
||||
if (!channel || channel.readyState !== 'open') return;
|
||||
callbacks.onStatus('calling-tool');
|
||||
let output: string;
|
||||
try {
|
||||
output = await callbacks.onToolCall(callId, name, argsJson);
|
||||
} catch {
|
||||
output = TOOL_CALL_FALLBACK_OUTPUT;
|
||||
}
|
||||
if (!isCurrent(attempt) || channel.readyState !== 'open') return;
|
||||
channel.send(JSON.stringify({
|
||||
event_id: `event_${Date.now()}`,
|
||||
type: 'conversation.item.create',
|
||||
item: { type: 'function_call_output', call_id: callId, output }
|
||||
}));
|
||||
channel.send(JSON.stringify({ event_id: `event_${Date.now()}`, type: 'response.create' }));
|
||||
callbacks.onStatus('connected');
|
||||
}).catch(() => {
|
||||
// 链内任何意外抛错不得毒化后续工具调用；本次调用软失败，对话继续
|
||||
});
|
||||
};
|
||||
|
||||
const handleEvent = (raw: string, attempt: number, provider: ProviderEventHandlers) => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
let event: RealtimeEvent;
|
||||
try {
|
||||
event = JSON.parse(raw) as RealtimeEvent;
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (event.type === 'session.created') provider.onSessionCreated();
|
||||
if (event.type === 'session.updated') provider.onSessionUpdated();
|
||||
if (event.type === 'input_audio_buffer.speech_started') callbacks.onStatus('listening');
|
||||
if (event.type === 'response.created') callbacks.onStatus('speaking');
|
||||
if (event.type === 'response.done') callbacks.onStatus('connected');
|
||||
if (event.type === 'conversation.item.input_audio_transcription.completed') {
|
||||
const transcript = trimTranscript(event.transcript);
|
||||
if (transcript) callbacks.onTraineeTranscript(transcript);
|
||||
}
|
||||
if (event.type === 'response.audio_transcript.done') {
|
||||
const transcript = trimTranscript(event.transcript);
|
||||
if (transcript) callbacks.onCustomerTranscript(transcript);
|
||||
}
|
||||
if (event.type === 'response.function_call_arguments.done') {
|
||||
handleToolCall(attempt, event);
|
||||
}
|
||||
if (event.type === 'error') {
|
||||
const error = new Error(event.error?.message || '实时陪练连接异常，请改用录音对练');
|
||||
callbacks.onError(error.message);
|
||||
provider.onSessionError(error);
|
||||
}
|
||||
};
|
||||
|
||||
const bindChannel = (channel: RTCDataChannel, attempt: number, provider: ProviderEventHandlers) => {
|
||||
channel.onmessage = (event) => handleEvent(String(event.data || ''), attempt, provider);
|
||||
channel.onerror = () => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
const error = new Error('实时陪练数据通道异常，请改用录音对练');
|
||||
callbacks.onError(error.message);
|
||||
provider.onSessionError(error);
|
||||
};
|
||||
};
|
||||
|
||||
const stop = () => {
|
||||
activeAttempt += 1;
|
||||
cancelSessionWait?.();
|
||||
cancelSessionWait = null;
|
||||
sessionWaitAttempt = null;
|
||||
toolCallChain = Promise.resolve();
|
||||
releaseResources();
|
||||
callbacks.onAudioPlaybackBlocked(false);
|
||||
callbacks.onStatus('idle');
|
||||
};
|
||||
|
||||
const terminate = (attempt: number, message: string) => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
callbacks.onError(message);
|
||||
stop();
|
||||
callbacks.onTerminated();
|
||||
};
|
||||
|
||||
const resumeAudio = async () => {
|
||||
if (!speaker) return false;
|
||||
try {
|
||||
await speaker.play();
|
||||
callbacks.onAudioPlaybackBlocked(false);
|
||||
return true;
|
||||
} catch {
|
||||
callbacks.onAudioPlaybackBlocked(true);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
const start = async (sessionUpdate: Record<string, unknown>, sessionId?: string) => {
|
||||
if (!isSupported()) throw new Error('当前设备不支持实时语音，请使用录音对练');
|
||||
stop();
|
||||
const attempt = ++activeAttempt;
|
||||
callbacks.onStatus('connecting');
|
||||
|
||||
let localMicrophone: MediaStream | null = null;
|
||||
let localConnection: RTCPeerConnection | null = null;
|
||||
let localChannel: RTCDataChannel | null = null;
|
||||
let localSpeaker: HTMLAudioElement | null = null;
|
||||
let readyTimeout: ReturnType<typeof setTimeout> | undefined;
|
||||
let disconnectTimeout: ReturnType<typeof setTimeout> | undefined;
|
||||
let sessionUpdateSent = false;
|
||||
let resolveReady: (() => void) | undefined;
|
||||
let rejectReady: ((error: Error) => void) | undefined;
|
||||
let sessionReady: Promise<void> | null = null;
|
||||
const releaseLocalResources = () => {
|
||||
if (disconnectTimeout) clearTimeout(disconnectTimeout);
|
||||
localChannel?.close();
|
||||
localMicrophone?.getTracks().forEach((track) => track.stop());
|
||||
if (localSpeaker) {
|
||||
localSpeaker.pause();
|
||||
localSpeaker.srcObject = null;
|
||||
return response.answerSdp;
|
||||
};
|
||||
|
||||
export const createRealtimePracticeController = (
|
||||
callbacks: RealtimePracticeCallbacks
|
||||
): RealtimePracticeController => {
|
||||
const engine = createRealtimeBrowserEngine(callbacks, exchangeRealtimeSdp);
|
||||
return {
|
||||
...engine,
|
||||
start: async (sessionUpdate, sessionId) => {
|
||||
if (isNativeAppRuntime()) {
|
||||
throw new Error('APP 实时语音应由 WebView 桥接启动');
|
||||
}
|
||||
localConnection?.close();
|
||||
};
|
||||
const ensureCurrent = () => {
|
||||
if (isCurrent(attempt)) return true;
|
||||
releaseLocalResources();
|
||||
return false;
|
||||
};
|
||||
|
||||
try {
|
||||
localMicrophone = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
if (!ensureCurrent()) return;
|
||||
microphone = localMicrophone;
|
||||
localConnection = new RTCPeerConnection({ iceServers: [] });
|
||||
connection = localConnection;
|
||||
const audioSender = localConnection.addTransceiver('audio', { direction: 'sendrecv' }).sender;
|
||||
const provider: ProviderEventHandlers = {
|
||||
onSessionCreated: () => {
|
||||
if (!isCurrent(attempt) || !localChannel || sessionUpdateSent) return;
|
||||
sessionUpdateSent = true;
|
||||
sendSessionUpdate(localChannel, sessionUpdate);
|
||||
},
|
||||
onSessionUpdated: () => {
|
||||
if (!isCurrent(attempt) || !sessionUpdateSent) return;
|
||||
const track = localMicrophone?.getAudioTracks()[0] || null;
|
||||
void audioSender.replaceTrack(track)
|
||||
.then(() => {
|
||||
if (isCurrent(attempt)) resolveReady?.();
|
||||
})
|
||||
.catch((error: unknown) => rejectReady?.(error instanceof Error ? error : new Error('实时陪练麦克风启动失败')));
|
||||
},
|
||||
onSessionError: (error) => {
|
||||
rejectReady?.(error);
|
||||
terminate(attempt, error.message);
|
||||
}
|
||||
};
|
||||
|
||||
localChannel = localConnection.createDataChannel('oai-events');
|
||||
controlChannel = localChannel;
|
||||
bindChannel(localChannel, attempt, provider);
|
||||
localConnection.ondatachannel = (event) => bindChannel(event.channel, attempt, provider);
|
||||
localConnection.ontrack = (event) => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
const stream = event.streams[0] || new MediaStream([event.track]);
|
||||
localSpeaker = new Audio();
|
||||
speaker = localSpeaker;
|
||||
localSpeaker.autoplay = true;
|
||||
localSpeaker.setAttribute('playsinline', 'true');
|
||||
localSpeaker.srcObject = stream;
|
||||
void localSpeaker.play()
|
||||
.then(() => {
|
||||
if (isCurrent(attempt)) callbacks.onAudioPlaybackBlocked(false);
|
||||
})
|
||||
.catch(() => {
|
||||
if (isCurrent(attempt)) callbacks.onAudioPlaybackBlocked(true);
|
||||
});
|
||||
};
|
||||
localConnection.onconnectionstatechange = () => {
|
||||
if (!isCurrent(attempt)) return;
|
||||
if (localConnection?.connectionState === 'failed' || localConnection?.connectionState === 'closed') {
|
||||
terminate(attempt, '实时陪练已断开，请改用录音对练');
|
||||
return;
|
||||
}
|
||||
if (localConnection?.connectionState === 'disconnected') {
|
||||
disconnectTimeout ??= setTimeout(() => {
|
||||
if (localConnection?.connectionState === 'disconnected') {
|
||||
terminate(attempt, '实时陪练已断开，请改用录音对练');
|
||||
}
|
||||
}, 3000);
|
||||
} else if (disconnectTimeout) {
|
||||
clearTimeout(disconnectTimeout);
|
||||
disconnectTimeout = undefined;
|
||||
}
|
||||
};
|
||||
|
||||
const offer = await localConnection.createOffer();
|
||||
if (!ensureCurrent()) return;
|
||||
await localConnection.setLocalDescription(offer);
|
||||
if (!ensureCurrent()) return;
|
||||
await waitForIceComplete(localConnection);
|
||||
if (!ensureCurrent()) return;
|
||||
const offerSdp = localConnection.localDescription?.sdp;
|
||||
if (!offerSdp) throw new Error('实时陪练连接请求创建失败');
|
||||
const response = await apiRequest<RealtimeSdpResponse>({
|
||||
url: '/api/train/practice/realtime/sdp',
|
||||
method: 'POST',
|
||||
data: { offerSdp: normalizeSdp(offerSdp), sessionId: sessionId || null },
|
||||
timeout: 30000
|
||||
});
|
||||
if (!ensureCurrent()) return;
|
||||
sessionReady = new Promise<void>((resolve, reject) => {
|
||||
resolveReady = resolve;
|
||||
rejectReady = reject;
|
||||
readyTimeout = setTimeout(() => reject(new Error('实时陪练会话初始化超时，请使用录音对练')), 15000);
|
||||
});
|
||||
void sessionReady.catch(() => undefined);
|
||||
sessionWaitAttempt = attempt;
|
||||
cancelSessionWait = () => {
|
||||
if (readyTimeout) clearTimeout(readyTimeout);
|
||||
rejectReady?.(new Error('实时陪练已结束'));
|
||||
};
|
||||
await localConnection.setRemoteDescription({ type: 'answer', sdp: normalizeSdp(response.answerSdp) });
|
||||
if (!ensureCurrent()) return;
|
||||
const waitForSessionReady = sessionReady;
|
||||
if (!waitForSessionReady) throw new Error('实时陪练会话初始化失败');
|
||||
await waitForSessionReady;
|
||||
if (!ensureCurrent()) return;
|
||||
callbacks.onStatus('connected');
|
||||
} catch (error) {
|
||||
if (!isCurrent(attempt)) {
|
||||
releaseLocalResources();
|
||||
return;
|
||||
}
|
||||
stop();
|
||||
throw error instanceof Error ? error : new Error('实时陪练连接失败，请改用录音对练');
|
||||
} finally {
|
||||
if (readyTimeout) clearTimeout(readyTimeout);
|
||||
if (sessionWaitAttempt === attempt) {
|
||||
sessionWaitAttempt = null;
|
||||
cancelSessionWait = null;
|
||||
}
|
||||
await engine.start(sessionUpdate, sessionId);
|
||||
}
|
||||
};
|
||||
|
||||
return { start, stop, resumeAudio };
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user