feat(practice): run realtime practice on App-Plus through renderjs
App-Plus has no WebRTC or getUserMedia in the logic layer, so realtime practice was H5-only and the native entry had to advertise a record-then- transcribe downgrade. Move the media/WebRTC engine into `realtime-browser-engine.js` and drive it from `RealtimePracticeAppBridge.vue`, whose renderjs script runs inside the system WebView where those APIs do exist. The logic layer keeps the authenticated calls: SDP exchange and `search_knowledge` tool invocation stay server-proxied, so the access token is never handed to the view layer. Both sides talk over the renderjs bridge only in session payloads the server already assembled. Verified against a real App build (`app-renderjs.js` holds RTCPeerConnection and getUserMedia; `app-service.js` holds the SDP endpoint, renderjs side has zero hits). Microphone permission, live transcription, remote playback, foreground/background switching and disconnect fallback still require Android and iOS device verification. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -485,6 +485,18 @@
|
||||
<button class="secondary-button" @click="openPracticeHistory">查看训练记录</button>
|
||||
<button class="primary-button tabbar-safe-action" :disabled="!canStartPractice || busy" @click="start(selectedScenarioId)">再练一次</button>
|
||||
</view>
|
||||
|
||||
<!-- #ifdef APP-PLUS -->
|
||||
<RealtimePracticeAppBridge
|
||||
ref="realtimeAppBridge"
|
||||
@status="handleRealtimeStatus"
|
||||
@trainee-transcript="handleRealtimeTraineeTranscript"
|
||||
@customer-transcript="handleRealtimeCustomerTranscript"
|
||||
@audio-playback-blocked="handleRealtimeAudioPlaybackBlocked"
|
||||
@terminated="handleRealtimeTerminated"
|
||||
@error="handleRealtimeError"
|
||||
/>
|
||||
<!-- #endif -->
|
||||
</view>
|
||||
</template>
|
||||
|
||||
@@ -492,6 +504,7 @@
|
||||
import { computed, nextTick, ref } from 'vue';
|
||||
import { onHide, onShow, onUnload } from '@dcloudio/uni-app';
|
||||
import ChatComposer from '@/components/chat/ChatComposer.vue';
|
||||
import RealtimePracticeAppBridge from '@/components/practice/RealtimePracticeAppBridge.vue';
|
||||
import type {
|
||||
PracticeFinishResponse,
|
||||
PracticePrepCard,
|
||||
@@ -513,7 +526,7 @@ import { searchKnowledge } from '@/services/knowledge';
|
||||
import { resetPageScroll } from '@/services/navigation';
|
||||
import { ensureEmployeePosition } from '@/services/position';
|
||||
import { chooseSpeechAudio, createSpeechPlaybackController, customerVoiceProfile, measureAudioDuration, MENTOR_VOICE_PROFILE, normalizeOssId, transcribeSpeechCapture, transcribeSpeechFile } from '@/services/speech';
|
||||
import { startSpeechCapture } from '@/services/speech-capture';
|
||||
import { isNativeAppRuntime, startSpeechCapture } from '@/services/speech-capture';
|
||||
import type { SpeechCapture, SpeechCaptureSession } from '@/services/speech-capture';
|
||||
import { createPageRequestScope, currentAccountKey, type RequestScopeTicket } from '@/services/request-scope';
|
||||
import { createRealtimePracticeController, createRealtimeSession, fetchRealtimePersonas, invokeRealtimeTool, type RealtimePersonaOption } from '@/services/realtime-practice';
|
||||
@@ -572,6 +585,11 @@ const realtimeAudioBlocked = ref(false);
|
||||
const realtimeSessionId = ref('');
|
||||
const realtimePersonaOptions = ref<RealtimePersonaOption[]>([]);
|
||||
const selectedRealtimePersonaId = ref(typeof storedRealtimePersona === 'string' ? storedRealtimePersona : '');
|
||||
const realtimeAppBridge = ref<{
|
||||
start: (sessionUpdate: Record<string, unknown>, sessionId?: string) => Promise<void>;
|
||||
stop: () => void;
|
||||
resumeAudio: () => Promise<boolean>;
|
||||
} | null>(null);
|
||||
const growthDetailsExpanded = ref(false);
|
||||
const assignments = ref<PracticeAssignment[]>([]);
|
||||
const practiceHistory = ref<PracticeRecord[]>([]);
|
||||
@@ -1010,34 +1028,40 @@ const appendTurn = (role: PracticeRole, text?: string, emotion = 0, synthesizeCu
|
||||
scrollThreadToBottom();
|
||||
};
|
||||
|
||||
const handleRealtimeStatus = (nextStatus: typeof realtimeStatus.value) => {
|
||||
realtimeStatus.value = nextStatus;
|
||||
};
|
||||
const handleRealtimeTraineeTranscript = (text: string) => {
|
||||
if (realtimeActive.value) appendTurn('trainee', text);
|
||||
};
|
||||
const handleRealtimeCustomerTranscript = (text: string) => {
|
||||
if (realtimeActive.value) appendTurn('customer', text, 0, false);
|
||||
};
|
||||
const handleRealtimeAudioPlaybackBlocked = (blocked: boolean) => {
|
||||
if (realtimeActive.value) realtimeAudioBlocked.value = blocked;
|
||||
};
|
||||
const handleRealtimeTerminated = () => {
|
||||
if (!realtimeActive.value) return;
|
||||
realtimeActive.value = false;
|
||||
realtimeStatus.value = 'idle';
|
||||
realtimeAudioBlocked.value = false;
|
||||
realtimeSessionId.value = '';
|
||||
sessionId.value = '';
|
||||
activeAssignmentId.value = undefined;
|
||||
status.value = 'idle';
|
||||
};
|
||||
const handleRealtimeError = (text: string) => {
|
||||
if (realtimeActive.value) message.value = text;
|
||||
};
|
||||
const realtimePractice = createRealtimePracticeController({
|
||||
onStatus: (nextStatus) => {
|
||||
realtimeStatus.value = nextStatus;
|
||||
},
|
||||
onTraineeTranscript: (text) => {
|
||||
if (realtimeActive.value) appendTurn('trainee', text);
|
||||
},
|
||||
onCustomerTranscript: (text) => {
|
||||
if (realtimeActive.value) appendTurn('customer', text, 0, false);
|
||||
},
|
||||
onAudioPlaybackBlocked: (blocked) => {
|
||||
if (realtimeActive.value) realtimeAudioBlocked.value = blocked;
|
||||
},
|
||||
onStatus: handleRealtimeStatus,
|
||||
onTraineeTranscript: handleRealtimeTraineeTranscript,
|
||||
onCustomerTranscript: handleRealtimeCustomerTranscript,
|
||||
onAudioPlaybackBlocked: handleRealtimeAudioPlaybackBlocked,
|
||||
onToolCall: (callId, name, argsJson) =>
|
||||
invokeRealtimeTool(realtimeSessionId.value, callId, name, argsJson).then((res) => res.output),
|
||||
onTerminated: () => {
|
||||
if (!realtimeActive.value) return;
|
||||
realtimeActive.value = false;
|
||||
realtimeStatus.value = 'idle';
|
||||
realtimeAudioBlocked.value = false;
|
||||
realtimeSessionId.value = '';
|
||||
sessionId.value = '';
|
||||
activeAssignmentId.value = undefined;
|
||||
status.value = 'idle';
|
||||
},
|
||||
onError: (text) => {
|
||||
if (realtimeActive.value) message.value = text;
|
||||
}
|
||||
onTerminated: handleRealtimeTerminated,
|
||||
onError: handleRealtimeError
|
||||
});
|
||||
|
||||
const selectRealtimePersona = (item: RealtimePersonaOption) => {
|
||||
@@ -1062,7 +1086,8 @@ const loadRealtimePersonas = async (request: RequestScopeTicket = requests.begin
|
||||
|
||||
const stopRealtime = () => {
|
||||
if (!realtimeActive.value && realtimeStatus.value === 'idle') return;
|
||||
realtimePractice.stop();
|
||||
if (isNativeAppRuntime()) realtimeAppBridge.value?.stop();
|
||||
else realtimePractice.stop();
|
||||
realtimeActive.value = false;
|
||||
realtimeStatus.value = 'idle';
|
||||
realtimeAudioBlocked.value = false;
|
||||
@@ -1093,7 +1118,9 @@ const startRealtime = async () => {
|
||||
if (!realtimeActive.value) return;
|
||||
realtimeSessionId.value = session.sessionId;
|
||||
activeScenarioName.value = session.personaName || '实时陪练';
|
||||
await realtimePractice.start(session.sessionUpdate, session.sessionId);
|
||||
const controller = isNativeAppRuntime() ? realtimeAppBridge.value : realtimePractice;
|
||||
if (!controller) throw new Error('实时语音模块尚未就绪,请稍后重试');
|
||||
await controller.start(session.sessionUpdate, session.sessionId);
|
||||
if (!realtimeActive.value) return;
|
||||
status.value = 'active';
|
||||
message.value = `已连接「${session.personaName}」,请直接开口说话。`;
|
||||
@@ -1107,7 +1134,8 @@ const startRealtime = async () => {
|
||||
};
|
||||
|
||||
const resumeRealtimeAudio = async () => {
|
||||
if (await realtimePractice.resumeAudio()) {
|
||||
const controller = isNativeAppRuntime() ? realtimeAppBridge.value : realtimePractice;
|
||||
if (controller && await controller.resumeAudio()) {
|
||||
message.value = '已恢复实时语音播放。';
|
||||
} else {
|
||||
message.value = '仍无法播放实时语音,请确认设备媒体权限后重试。';
|
||||
|
||||
Reference in New Issue
Block a user