feat(practice): run realtime practice on App-Plus through renderjs

App-Plus has no WebRTC or getUserMedia in the logic layer, so realtime
practice was H5-only and the native entry had to advertise a record-then-
transcribe downgrade.

Move the media/WebRTC engine into `realtime-browser-engine.js` and drive it
from `RealtimePracticeAppBridge.vue`, whose renderjs script runs inside the
system WebView where those APIs do exist. The logic layer keeps the
authenticated calls: SDP exchange and `search_knowledge` tool invocation stay
server-proxied, so the access token is never handed to the view layer. Both
sides talk over the renderjs bridge only in session payloads the server
already assembled.

Verified against a real App build (`app-renderjs.js` holds RTCPeerConnection
and getUserMedia; `app-service.js` holds the SDP endpoint, renderjs side has
zero hits). Microphone permission, live transcription, remote playback,
foreground/background switching and disconnect fallback still require Android
and iOS device verification.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-07-25 23:58:10 +08:00
co-authored by Claude Opus 5
parent 3055d2bec9
commit 5cf7ce5847
10 changed files with 673 additions and 390 deletions
+57 -29
View File
@@ -485,6 +485,18 @@
<button class="secondary-button" @click="openPracticeHistory">查看训练记录</button>
<button class="primary-button tabbar-safe-action" :disabled="!canStartPractice || busy" @click="start(selectedScenarioId)">再练一次</button>
</view>
<!-- #ifdef APP-PLUS -->
<RealtimePracticeAppBridge
ref="realtimeAppBridge"
@status="handleRealtimeStatus"
@trainee-transcript="handleRealtimeTraineeTranscript"
@customer-transcript="handleRealtimeCustomerTranscript"
@audio-playback-blocked="handleRealtimeAudioPlaybackBlocked"
@terminated="handleRealtimeTerminated"
@error="handleRealtimeError"
/>
<!-- #endif -->
</view>
</template>
@@ -492,6 +504,7 @@
import { computed, nextTick, ref } from 'vue';
import { onHide, onShow, onUnload } from '@dcloudio/uni-app';
import ChatComposer from '@/components/chat/ChatComposer.vue';
import RealtimePracticeAppBridge from '@/components/practice/RealtimePracticeAppBridge.vue';
import type {
PracticeFinishResponse,
PracticePrepCard,
@@ -513,7 +526,7 @@ import { searchKnowledge } from '@/services/knowledge';
import { resetPageScroll } from '@/services/navigation';
import { ensureEmployeePosition } from '@/services/position';
import { chooseSpeechAudio, createSpeechPlaybackController, customerVoiceProfile, measureAudioDuration, MENTOR_VOICE_PROFILE, normalizeOssId, transcribeSpeechCapture, transcribeSpeechFile } from '@/services/speech';
import { startSpeechCapture } from '@/services/speech-capture';
import { isNativeAppRuntime, startSpeechCapture } from '@/services/speech-capture';
import type { SpeechCapture, SpeechCaptureSession } from '@/services/speech-capture';
import { createPageRequestScope, currentAccountKey, type RequestScopeTicket } from '@/services/request-scope';
import { createRealtimePracticeController, createRealtimeSession, fetchRealtimePersonas, invokeRealtimeTool, type RealtimePersonaOption } from '@/services/realtime-practice';
@@ -572,6 +585,11 @@ const realtimeAudioBlocked = ref(false);
const realtimeSessionId = ref('');
const realtimePersonaOptions = ref<RealtimePersonaOption[]>([]);
const selectedRealtimePersonaId = ref(typeof storedRealtimePersona === 'string' ? storedRealtimePersona : '');
const realtimeAppBridge = ref<{
start: (sessionUpdate: Record<string, unknown>, sessionId?: string) => Promise<void>;
stop: () => void;
resumeAudio: () => Promise<boolean>;
} | null>(null);
const growthDetailsExpanded = ref(false);
const assignments = ref<PracticeAssignment[]>([]);
const practiceHistory = ref<PracticeRecord[]>([]);
@@ -1010,34 +1028,40 @@ const appendTurn = (role: PracticeRole, text?: string, emotion = 0, synthesizeCu
scrollThreadToBottom();
};
const handleRealtimeStatus = (nextStatus: typeof realtimeStatus.value) => {
realtimeStatus.value = nextStatus;
};
const handleRealtimeTraineeTranscript = (text: string) => {
if (realtimeActive.value) appendTurn('trainee', text);
};
const handleRealtimeCustomerTranscript = (text: string) => {
if (realtimeActive.value) appendTurn('customer', text, 0, false);
};
const handleRealtimeAudioPlaybackBlocked = (blocked: boolean) => {
if (realtimeActive.value) realtimeAudioBlocked.value = blocked;
};
const handleRealtimeTerminated = () => {
if (!realtimeActive.value) return;
realtimeActive.value = false;
realtimeStatus.value = 'idle';
realtimeAudioBlocked.value = false;
realtimeSessionId.value = '';
sessionId.value = '';
activeAssignmentId.value = undefined;
status.value = 'idle';
};
const handleRealtimeError = (text: string) => {
if (realtimeActive.value) message.value = text;
};
const realtimePractice = createRealtimePracticeController({
onStatus: (nextStatus) => {
realtimeStatus.value = nextStatus;
},
onTraineeTranscript: (text) => {
if (realtimeActive.value) appendTurn('trainee', text);
},
onCustomerTranscript: (text) => {
if (realtimeActive.value) appendTurn('customer', text, 0, false);
},
onAudioPlaybackBlocked: (blocked) => {
if (realtimeActive.value) realtimeAudioBlocked.value = blocked;
},
onStatus: handleRealtimeStatus,
onTraineeTranscript: handleRealtimeTraineeTranscript,
onCustomerTranscript: handleRealtimeCustomerTranscript,
onAudioPlaybackBlocked: handleRealtimeAudioPlaybackBlocked,
onToolCall: (callId, name, argsJson) =>
invokeRealtimeTool(realtimeSessionId.value, callId, name, argsJson).then((res) => res.output),
onTerminated: () => {
if (!realtimeActive.value) return;
realtimeActive.value = false;
realtimeStatus.value = 'idle';
realtimeAudioBlocked.value = false;
realtimeSessionId.value = '';
sessionId.value = '';
activeAssignmentId.value = undefined;
status.value = 'idle';
},
onError: (text) => {
if (realtimeActive.value) message.value = text;
}
onTerminated: handleRealtimeTerminated,
onError: handleRealtimeError
});
const selectRealtimePersona = (item: RealtimePersonaOption) => {
@@ -1062,7 +1086,8 @@ const loadRealtimePersonas = async (request: RequestScopeTicket = requests.begin
const stopRealtime = () => {
if (!realtimeActive.value && realtimeStatus.value === 'idle') return;
realtimePractice.stop();
if (isNativeAppRuntime()) realtimeAppBridge.value?.stop();
else realtimePractice.stop();
realtimeActive.value = false;
realtimeStatus.value = 'idle';
realtimeAudioBlocked.value = false;
@@ -1093,7 +1118,9 @@ const startRealtime = async () => {
if (!realtimeActive.value) return;
realtimeSessionId.value = session.sessionId;
activeScenarioName.value = session.personaName || '实时陪练';
await realtimePractice.start(session.sessionUpdate, session.sessionId);
const controller = isNativeAppRuntime() ? realtimeAppBridge.value : realtimePractice;
if (!controller) throw new Error('实时语音模块尚未就绪,请稍后重试');
await controller.start(session.sessionUpdate, session.sessionId);
if (!realtimeActive.value) return;
status.value = 'active';
message.value = `已连接「${session.personaName}」,请直接开口说话。`;
@@ -1107,7 +1134,8 @@ const startRealtime = async () => {
};
const resumeRealtimeAudio = async () => {
if (await realtimePractice.resumeAudio()) {
const controller = isNativeAppRuntime() ? realtimeAppBridge.value : realtimePractice;
if (controller && await controller.resumeAudio()) {
message.value = '已恢复实时语音播放。';
} else {
message.value = '仍无法播放实时语音,请确认设备媒体权限后重试。';