From 4ef388873b7ef02c314d7c6076861b828a784fae Mon Sep 17 00:00:00 2001 From: let5sne Date: Thu, 9 Jul 2026 13:41:48 +0800 Subject: [PATCH] feat(mobile): add voice transcription to practice --- docs/20260708/uni-app用户侧前端重建计划.md | 1 + mobile-uni/src/pages/user/practice/index.vue | 158 ++++++++++++++++++- mobile-uni/src/services/navigation.ts | 6 + mobile-uni/src/services/practice.ts | 2 + mobile-uni/src/services/speech.ts | 67 ++++++++ mobile-uni/src/types/api.ts | 13 ++ 6 files changed, 245 insertions(+), 2 deletions(-) create mode 100644 mobile-uni/src/services/speech.ts diff --git a/docs/20260708/uni-app用户侧前端重建计划.md b/docs/20260708/uni-app用户侧前端重建计划.md index 6006ad16..65bb14ec 100644 --- a/docs/20260708/uni-app用户侧前端重建计划.md +++ b/docs/20260708/uni-app用户侧前端重建计划.md @@ -226,6 +226,7 @@ mobile-uni/ - 员工今日页“今日安排”已优先回读真实 `/api/aihr/mobile/practice/assignments`;有待训练专项或每日三题时覆盖公开 home seed,真实浏览器已验证 `13900000998` 首页显示当天 daily 任务并可跳到练习页。 - 员工今日页到练习页已补任务上下文传递:首页点“开始”会把当前 assignment 带到练习页,daily 任务高亮并提示填写,主管专项可直接启动,减少跳页后的二次选择。 - 移动端 hash 路由滚动复位已补:员工/候选人/主管/登录/建设中页面激活时回到顶部,真实浏览器验证从长页切回今日页后 `scrollY=0`,避免首屏被旧滚动位置截断。 +- 员工练习页已补语音输入入口:回复区支持浏览器录音转写和选择音频转写,ASR 返回文本进入原提交框且音频 URL/OssId 随 turn 提交供主管复盘回放;真实浏览器验证按钮在 390px 手机视口不换行,文本 turn 回归可进入第 2 轮。 ## 7. 风险 diff --git a/mobile-uni/src/pages/user/practice/index.vue b/mobile-uni/src/pages/user/practice/index.vue index e2ffe590..d9b1ee94 100644 --- a/mobile-uni/src/pages/user/practice/index.vue +++ b/mobile-uni/src/pages/user/practice/index.vue @@ -126,6 +126,12 @@ placeholder="先承接业主情绪，再说明处理动作和反馈时间。" /> + + @@ -134,6 +140,7 @@ {{ status === 'submitting' ? '提交中' : '提交回应' }} + 已附带语音回放 @@ -170,6 +177,7 @@ import { answerDailyDrill, finishPractice, getPracticeAssignments, getPracticePr import { isLoggedIn, rememberLoginRedirect } from '@/services/auth'; import { searchKnowledge } from '@/services/knowledge'; import { resetPageScroll } from '@/services/navigation'; +import { chooseSpeechAudio, transcribeSpeechBlob, transcribeSpeechFile } from '@/services/speech'; type PracticeStatus = 'idle' | 'starting' | 'active' | 'submitting' | 'finished'; @@ -203,6 +211,12 @@ const answeringId = ref(null); const activeAssignmentId = ref(); const highlightedAssignmentId = ref(null); const loggedIn = ref(false); +const recording = ref(false); +const transcribing = ref(false); +const draftAudioUrl = ref(''); +const draftAudioOssId = ref(); +const mediaRecorder = ref(null); +const mediaStream = ref(null); const busy = computed(() => status.value === 'starting' || status.value === 'submitting'); const canStartPractice = computed(() => status.value === 'idle' || status.value === 'finished'); const hasTraineeReply = computed(() => turns.value.some((item) => item.role === 'trainee')); @@ -237,6 +251,7 @@ const refreshAuthState = () => { }; const clearPracticeData = () => { + cancelRecording(); assignments.value = []; drillAnswers.value = {}; prepCard.value = null; @@ -261,6 +276,8 @@ const clearPracticeData = () => { answeringId.value = null; activeAssignmentId.value = undefined; highlightedAssignmentId.value = null; + transcribing.value = false; + clearDraftAudio(); }; const requireLogin = () => { @@ -276,6 +293,117 @@ const appendTurn = (role: PracticeRole, text?: string) => { turns.value.push({ role, text }); }; +const clearDraftAudio = () => { + draftAudioUrl.value = ''; + draftAudioOssId.value = undefined; +}; + +const stopMediaTracks = () => { + mediaStream.value?.getTracks().forEach((track) => track.stop()); + mediaStream.value = null; +}; + +const cancelRecording = () => { + const recorder = mediaRecorder.value; + if (recorder && recorder.state !== 'inactive') { + recorder.ondataavailable = null; + recorder.onstop = null; + recorder.stop(); + } + mediaRecorder.value = null; + stopMediaTracks(); + recording.value = false; +}; + +const applyAsrResult = (result: { text?: string; audioUrl?: string; ossId?: number | string }) => { + const text = (result.text || '').trim(); + if (!text) throw new Error('语音没有识别到文字'); + const current = draft.value.trim(); + draft.value = current ? `${current}\n${text}` : text; + draftAudioUrl.value = result.audioUrl || ''; + const ossId = Number(result.ossId); + draftAudioOssId.value = Number.isFinite(ossId) ? ossId : undefined; + message.value = '语音已转成文字，请确认后提交'; +}; + +const transcribeBlob = async (blob: Blob) => { + if (transcribing.value) return; + transcribing.value = true; + message.value = '语音转写中'; + try { + applyAsrResult(await transcribeSpeechBlob(blob)); + } catch (error) { + handleError(error, '语音转写失败'); + } finally { + transcribing.value = false; + } +}; + +const pickAudio = async () => { + if (busy.value || recording.value || transcribing.value) return; + if (!requireLogin()) return; + transcribing.value = true; + message.value = '语音转写中'; + try { + const file = await chooseSpeechAudio(); + applyAsrResult(await transcribeSpeechFile(file)); + } catch (error) { + handleError(error, '语音转写失败'); + } finally { + transcribing.value = false; + } +}; + +const startRecording = async () => { + if (busy.value || transcribing.value) return; + if (!requireLogin()) return; + if (typeof navigator === 'undefined' || !navigator.mediaDevices?.getUserMedia || typeof MediaRecorder === 'undefined') { + await pickAudio(); + return; + } + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const recorder = new MediaRecorder(stream); + const chunks: Blob[] = []; + mediaStream.value = stream; + mediaRecorder.value = recorder; + recorder.ondataavailable = (event) => { + if (event.data?.size) chunks.push(event.data); + }; + recorder.onstop = () => { + recording.value = false; + mediaRecorder.value = null; + stopMediaTracks(); + const blob = new Blob(chunks, { type: recorder.mimeType || 'audio/webm' }); + if (!blob.size) { + message.value = '未录到音频，请改用文字输入'; + return; + } + void transcribeBlob(blob); + }; + recorder.start(); + recording.value = true; + message.value = '正在录音'; + } catch (error) { + await pickAudio(); + } +}; + +const stopRecording = () => { + const recorder = mediaRecorder.value; + if (!recording.value || !recorder || recorder.state === 'inactive') return; + message.value = '录音结束，正在转写'; + recorder.stop(); +}; + +const toggleRecording = () => { + if (recording.value) { + stopRecording(); + return; + } + void startRecording(); +}; + const handleError = (error: unknown, fallback: string) => { const text = error instanceof Error ? error.message : fallback; refreshAuthState(); @@ -386,6 +514,7 @@ const start = async (scenarioId = defaultScenarioId, assignmentId?: number) => { result.value = null; turns.value = []; draft.value = ''; + clearDraftAudio(); helpAnswer.value = ''; helpRefs.value = []; lastTraineeText.value = ''; @@ -482,7 +611,10 @@ const submit = async () => { } status.value = 'submitting'; message.value = ''; + const audioUrl = draftAudioUrl.value || undefined; + const audioOssId = draftAudioOssId.value; draft.value = ''; + clearDraftAudio(); const retryingRedFlag = redFlagActive.value && lastCoachRoundIndex.value === roundIndex.value; if (retryingRedFlag) discardLastCoachAttempt(); appendTurn('trainee', text); @@ -491,7 +623,9 @@ const submit = async () => { const data = await submitPracticeTurn({ sessionId: sessionId.value, roundIndex: submittedRound, - traineeText: text + traineeText: text, + traineeAudioUrl: audioUrl, + traineeAudioOssId: audioOssId }); lastTraineeText.value = text; lastCoachRoundIndex.value = submittedRound; @@ -513,6 +647,9 @@ const submit = async () => { status.value = 'active'; } catch (error) { status.value = 'active'; + draft.value = text; + draftAudioUrl.value = audioUrl || ''; + draftAudioOssId.value = audioOssId; handleError(error, '提交失败'); } }; @@ -699,10 +836,27 @@ onShow(() => { .reply-actions { display: flex; align-items: center; - justify-content: space-between; gap: 10px; } +.reply-actions { + flex-wrap: wrap; +} + +.reply-actions button { + min-height: 48px; + min-width: 92px; + flex: 1; + white-space: nowrap; + word-break: keep-all; +} + +.voice-state { + color: #2563eb; + font-size: 13px; + font-weight: 800; +} + .round-label { color: #64748b; font-size: 13px; diff --git a/mobile-uni/src/services/navigation.ts b/mobile-uni/src/services/navigation.ts index 48ac447c..d19949fe 100644 --- a/mobile-uni/src/services/navigation.ts +++ b/mobile-uni/src/services/navigation.ts @@ -1,9 +1,15 @@ export const resetPageScroll = () => { const scrollTop = () => { uni.pageScrollTo({ scrollTop: 0, duration: 0 }); + if (typeof window !== 'undefined') { + window.scrollTo({ top: 0, left: 0, behavior: 'auto' }); + document.documentElement.scrollTop = 0; + document.body.scrollTop = 0; + } }; scrollTop(); setTimeout(scrollTop, 0); setTimeout(scrollTop, 80); + setTimeout(scrollTop, 240); }; diff --git a/mobile-uni/src/services/practice.ts b/mobile-uni/src/services/practice.ts index b63d4ef9..706a7585 100644 --- a/mobile-uni/src/services/practice.ts +++ b/mobile-uni/src/services/practice.ts @@ -50,6 +50,8 @@ export const submitPracticeTurn = (params: { sessionId: string; roundIndex: number; traineeText: string; + traineeAudioUrl?: string; + traineeAudioOssId?: number; regenerate?: boolean; style?: string; }) => diff --git a/mobile-uni/src/services/speech.ts b/mobile-uni/src/services/speech.ts new file mode 100644 index 00000000..60c93736 --- /dev/null +++ b/mobile-uni/src/services/speech.ts @@ -0,0 +1,67 @@ +import type { AsrResponse, SpeechSelectedFile } from '@/types/api'; +import { apiUrl, authHeaders, readTextPayload } from './api'; + +const audioExtensions = ['.mp3', '.wav', '.m4a', '.webm', '.aac', '.ogg']; + +export const chooseSpeechAudio = () => + new Promise((resolve, reject) => { + uni.chooseFile({ + count: 1, + type: 'all', + extension: audioExtensions, + success: (result) => { + const tempFiles = Array.isArray(result.tempFiles) ? result.tempFiles : []; + const file = (tempFiles[0] || {}) as Partial; + const path = file.path || result.tempFilePaths?.[0] || ''; + if (!path) { + reject(new Error('未获取到音频文件')); + return; + } + resolve({ + path, + name: file.name || path.split('/').pop() || 'practice-audio', + size: Number(file.size || 0) + }); + }, + fail: () => reject(new Error('未选择音频')) + }); + }); + +export const transcribeSpeechFile = (file: SpeechSelectedFile) => + new Promise((resolve, reject) => { + uni.uploadFile({ + url: apiUrl('/api/ai/asr'), + filePath: file.path, + name: 'file', + header: authHeaders(false), + success: (response) => { + try { + resolve(readTextPayload(response.statusCode, response.data)); + } catch (error) { + reject(error); + } + }, + fail: () => reject(new Error('语音上传失败')) + }); + }); + +export const transcribeSpeechBlob = (blob: Blob, filename = 'practice-audio.webm') => + new Promise((resolve, reject) => { + const form = new FormData(); + form.append('file', blob, filename); + + const xhr = new XMLHttpRequest(); + xhr.open('POST', apiUrl('/api/ai/asr')); + xhr.timeout = 45000; + Object.entries(authHeaders(false)).forEach(([key, value]) => xhr.setRequestHeader(key, value)); + xhr.onload = () => { + try { + resolve(readTextPayload(xhr.status, xhr.responseText || '{}')); + } catch (error) { + reject(error); + } + }; + xhr.onerror = () => reject(new Error('语音上传失败')); + xhr.ontimeout = () => reject(new Error('语音转写超时')); + xhr.send(form); + }); diff --git a/mobile-uni/src/types/api.ts b/mobile-uni/src/types/api.ts index e9912fa4..ee335b5f 100644 --- a/mobile-uni/src/types/api.ts +++ b/mobile-uni/src/types/api.ts @@ -145,6 +145,13 @@ export interface PracticeFinishResponse { trustCurve?: number[]; } +export interface AsrResponse { + text: string; + source: string; + ossId?: number | string; + audioUrl?: string; +} + export interface PracticeRecord { id?: number; sessionId?: string; @@ -255,6 +262,12 @@ export interface CaseSelectedFile { size: number; } +export interface SpeechSelectedFile { + path: string; + name: string; + size: number; +} + export interface CaseTag { text: string; type: string;