admin/prop-ai-hr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add qwen realtime practice beta

Browse Source
This commit is contained in:
admin
2026-07-22 09:29:39 +08:00
parent b7dd1bca27
commit 87772cc64c
7 changed files with 712 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/controller/AihrPracticeController.java
+16
View File
@@ -12,6 +12,8 @@ import org.dromara.aihr.domain.AihrPracticeDto.CalibrationResponse;
import org.dromara.aihr.domain.AihrPracticeDto.FinishRequest;
import org.dromara.aihr.domain.AihrPracticeDto.FinishResponse;
import org.dromara.aihr.domain.AihrPracticeDto.PrepCardResponse;
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeSdpRequest;
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeSdpResponse;
import org.dromara.aihr.domain.AihrPracticeDto.RubricResponse;
import org.dromara.aihr.domain.AihrPracticeDto.RubricAdminResponse;
import org.dromara.aihr.domain.AihrPracticeDto.RubricEnabledRequest;
@@ -24,6 +26,7 @@ import org.dromara.aihr.domain.AihrPracticeDto.StartResponse;
import org.dromara.aihr.domain.AihrPracticeDto.TurnRequest;
import org.dromara.aihr.domain.AihrPracticeDto.TurnResponse;
import org.dromara.aihr.service.AihrPracticeSeedService;
import org.dromara.aihr.service.AihrRealtimePracticeService;
import org.dromara.common.core.constant.TenantConstants;
import org.dromara.common.core.domain.R;
import org.dromara.common.core.domain.model.LoginUser;
@@ -57,6 +60,7 @@ public class AihrPracticeController {
private static final String HR_OPERATOR_ROLE = "hr_operator";
private final AihrPracticeSeedService practiceSeedService;
private final AihrRealtimePracticeService realtimePracticeService;
@GetMapping("/scenarios")
public R<List<ScenarioResponse>> scenarios(@RequestParam(required = false) String position,
@@ -129,6 +133,18 @@ public class AihrPracticeController {
return R.ok(practiceSeedService.start(bindMobileStartIdentity(request)));
}
/**
* Only proxies the WebRTC SDP handshake. Media flows directly between the H5 client and Model Studio.
*/
@PostMapping("/realtime/sdp")
public R<RealtimeSdpResponse> realtimeSdp(@RequestBody RealtimeSdpRequest request) {
String username = currentAppUsername();
if (username.isBlank()) {
return R.fail("请使用员工端账号开启实时陪练");
}
return R.ok(realtimePracticeService.exchangeSdp(request, username));
}
@PostMapping("/turn")
public R<TurnResponse> turn(@RequestBody TurnRequest request) {
return R.ok(practiceSeedService.turn(request, currentAppUsername(), currentAppUserId()));
backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/domain/AihrPracticeDto.java
+6
View File
@@ -107,6 +107,12 @@ public final class AihrPracticeDto {
public record PrepCardResponse(List<String> keyPoints, List<String> redLines, List<String> scripts, String promptVersion) {
}
public record RealtimeSdpRequest(String offerSdp) {
}
public record RealtimeSdpResponse(String answerSdp, String model) {
}
public record TurnRequest(String sessionId, Integer roundIndex, String traineeText, String traineeAudioUrl, Long traineeAudioOssId, Boolean regenerate, String style) {
}
backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/service/AihrRealtimePracticeService.java
+134
View File
@@ -0,0 +1,134 @@
package org.dromara.aihr.service;
import lombok.extern.slf4j.Slf4j;
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeSdpRequest;
import org.dromara.aihr.domain.AihrPracticeDto.RealtimeSdpResponse;
import org.dromara.common.core.exception.ServiceException;
import org.dromara.common.redis.utils.RedisUtils;
import org.dromara.common.satoken.utils.LoginHelper;
import org.redisson.api.RateType;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.net.URI;
import java.net.URLEncoder;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.Locale;
/**
* Minimal WebRTC signaling boundary for the H5 realtime-practice beta.
* Audio is deliberately not proxied or persisted here.
*/
@Service
@Slf4j
public class AihrRealtimePracticeService {
private static final int MAX_SDP_CHARS = 64 * 1024;
private static final HttpClient HTTP_CLIENT = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(10))
.build();
@Value("${aihr.realtime-practice.enabled:${AIHR_REALTIME_PRACTICE_ENABLED:false}}")
private boolean enabled;
@Value("${aihr.realtime-practice.endpoint:${AIHR_QWEN_REALTIME_ENDPOINT:}}")
private String endpoint;
@Value("${aihr.realtime-practice.api-key:${AIHR_QWEN_REALTIME_API_KEY:}}")
private String apiKey;
@Value("${aihr.realtime-practice.model:${AIHR_QWEN_REALTIME_MODEL:qwen3.5-omni-flash-realtime}}")
private String model;
public RealtimeSdpResponse exchangeSdp(RealtimeSdpRequest request, String username) {
if (!enabled) {
throw new ServiceException("实时陪练尚未启用，请使用录音对练");
}
String offer = request == null || request.offerSdp() == null ? "" : request.offerSdp();
String verifiedOffer = offer.trim();
if (verifiedOffer.isBlank() || offer.length() > MAX_SDP_CHARS || !verifiedOffer.contains("m=audio")) {
throw new ServiceException("实时陪练连接请求无效");
}
String account = username == null ? "" : username.trim();
if (account.isBlank()) {
throw new ServiceException("请使用员工端账号开启实时陪练");
}
enforceRateLimit(account);
URI signalingUri = signalingUri(endpoint, model);
if (apiKey == null || apiKey.isBlank()) {
throw new ServiceException("实时陪练尚未完成模型配置，请使用录音对练");
}
try {
HttpRequest httpRequest = HttpRequest.newBuilder()
.uri(signalingUri)
.timeout(Duration.ofSeconds(20))
.header("Content-Type", "application/sdp")
.header("Authorization", "Bearer " + apiKey.trim())
.POST(HttpRequest.BodyPublishers.ofString(offer, StandardCharsets.UTF_8))
.build();
HttpResponse<String> response = HTTP_CLIENT.send(httpRequest, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8));
String answer = response.body() == null ? "" : response.body();
String verifiedAnswer = answer.trim();
if (response.statusCode() != 200 || verifiedAnswer.isBlank() || answer.length() > MAX_SDP_CHARS || !verifiedAnswer.contains("m=audio")) {
log.warn("qwen realtime SDP exchange failed: status={}", response.statusCode());
throw new ServiceException("实时陪练暂时无法连接，请使用录音对练");
}
return new RealtimeSdpResponse(answer, normalizedModel(model));
} catch (IOException e) {
log.warn("qwen realtime SDP connection failed", e);
throw new ServiceException("实时陪练网络连接失败，请使用录音对练");
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new ServiceException("实时陪练连接被中断，请重试");
}
}
static URI signalingUri(String configuredEndpoint, String configuredModel) {
String value = configuredEndpoint == null ? "" : configuredEndpoint.trim();
URI base;
try {
base = URI.create(value);
} catch (IllegalArgumentException e) {
throw new ServiceException("实时陪练模型地址配置无效");
}
String host = base.getHost() == null ? "" : base.getHost().toLowerCase(Locale.ROOT);
if (!"https".equalsIgnoreCase(base.getScheme()) || !host.endsWith(".maas.aliyuncs.com")
|| base.getUserInfo() != null || base.getQuery() != null || base.getFragment() != null) {
throw new ServiceException("实时陪练模型地址配置无效");
}
String basePath = base.getPath() == null ? "" : base.getPath().replaceAll("/+$", "");
if (!basePath.isEmpty()) {
throw new ServiceException("实时陪练模型地址配置无效");
}
String encodedModel = URLEncoder.encode(normalizedModel(configuredModel), StandardCharsets.UTF_8);
return URI.create(base.getScheme() + "://" + host + "/api/v1/webrtc/realtime?model=" + encodedModel);
}
private static String normalizedModel(String configuredModel) {
String value = configuredModel == null ? "" : configuredModel.trim();
if (!value.matches("qwen3\\.5-omni-(flash|plus)-realtime")) {
throw new ServiceException("实时陪练模型配置无效");
}
return value;
}
private static void enforceRateLimit(String username) {
String key = "aihr:practice:realtime:sdp:" + LoginHelper.getTenantId() + ":" + username;
try {
if (RedisUtils.rateLimiter(key, RateType.OVERALL, 3, 60) < 0) {
throw new ServiceException("实时陪练连接过于频繁，请稍后重试", 429);
}
} catch (ServiceException known) {
throw known;
} catch (RuntimeException unavailable) {
log.warn("realtime practice rate limiter unavailable");
throw new ServiceException("实时陪练限流暂时不可用，请稍后重试", 503);
}
}
}
backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/service/AihrRealtimePracticeServiceTest.java
+29
View File
@@ -0,0 +1,29 @@
package org.dromara.aihr.service;
import org.dromara.common.core.exception.ServiceException;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
class AihrRealtimePracticeServiceTest {
@Test
@Tag("dev")
void signalingUriOnlyAllowsOfficialModelStudioHostsAndRealtimeModels() {
assertEquals(
"https://workspace.cn-beijing.maas.aliyuncs.com/api/v1/webrtc/realtime?model=qwen3.5-omni-flash-realtime",
AihrRealtimePracticeService.signalingUri(
"https://workspace.cn-beijing.maas.aliyuncs.com",
"qwen3.5-omni-flash-realtime"
).toString()
);
assertThrows(ServiceException.class, () -> AihrRealtimePracticeService.signalingUri(
"https://example.com", "qwen3.5-omni-flash-realtime"
));
assertThrows(ServiceException.class, () -> AihrRealtimePracticeService.signalingUri(
"https://workspace.cn-beijing.maas.aliyuncs.com", "other-model"
));
}
}
mobile-uni/src/pages/user/practice/index.vue
+174 -4
View File
@@ -51,6 +51,15 @@
>
{{ status === 'finished' ? '再练一次' : '我已准备好，开始练习' }}
</button>
<button
v-if="status === 'idle' || status === 'finished'"
class="secondary-button realtime-start-button"
:disabled="busy"
@click="startRealtime"
>
实时陪练 Beta
</button>
<text class="realtime-beta-note">自然打断、边听边说；本次体验不计入训练成绩。</text>
</view>
<view v-if="loggedIn && practiceView === 'prep'" class="panel assignment-panel recent-panel">
@@ -159,7 +168,7 @@
<view class="chat-head">
<view class="chat-head-row">
<text class="chat-title">{{ activeScenarioName }}</text>
<text class="chat-context">第 {{ Math.min(roundIndex + 1, 3) }}/3 轮</text>
<text class="chat-context">{{ realtimeActive ? `实时 · ${realtimeStatusLabel}` : `第 ${Math.min(roundIndex + 1, 3)}/3 轮` }}</text>
</view>
<view v-if="hasCoachState" class="coach-strip">
<view class="coach-metric">
@@ -253,7 +262,18 @@
</view>
</scroll-view>
<view v-if="canAdjustCoach" class="style-chip-row">
<view v-if="realtimeActive" class="realtime-practice-panel">
<view>
<text class="realtime-practice-title">实时陪练 Beta</text>
<text class="realtime-practice-copy">直接说话即可；本次内容仅供即时练习，不写入训练成绩或主管复盘。</text>
</view>
<view class="realtime-practice-actions">
<button v-if="realtimeAudioBlocked" class="secondary-button realtime-resume-button" @click="resumeRealtimeAudio">点击播放</button>
<button class="secondary-button realtime-stop-button" @click="stopRealtime">结束体验</button>
</view>
</view>
<view v-if="!realtimeActive && canAdjustCoach" class="style-chip-row">
<button
v-for="item in styleOptions"
:key="item.value"
@@ -266,6 +286,7 @@
</view>
<ChatComposer
v-if="!realtimeActive"
ref="composerRef"
:disabled="busy"
:transcribing="transcribing"
@@ -366,6 +387,7 @@ import { resetPageScroll } from '@/services/navigation';
import { ensureEmployeePosition } from '@/services/position';
import { chooseSpeechAudio, createSpeechPlaybackController, customerVoiceProfile, measureAudioDuration, MENTOR_VOICE_PROFILE, normalizeOssId, transcribeSpeechBlob, transcribeSpeechFile } from '@/services/speech';
import { createPageRequestScope, currentAccountKey, type RequestScopeTicket } from '@/services/request-scope';
import { createRealtimePracticeController } from '@/services/realtime-practice';
import { stripInternalCodes } from '@/services/text';
type PracticeStatus = 'idle' | 'starting' | 'active' | 'submitting' | 'finished';
@@ -408,6 +430,9 @@ const adjustingStyle = ref('');
const emotionScore = ref(0);
const trustScore = ref(0);
const redFlagActive = ref(false);
const realtimeActive = ref(false);
const realtimeStatus = ref<'idle' | 'connecting' | 'connected' | 'listening' | 'speaking'>('idle');
const realtimeAudioBlocked = ref(false);
const assignments = ref<PracticeAssignment[]>([]);
const practiceHistory = ref<PracticeRecord[]>([]);
const drillAnswers = ref<Record<number, string>>({});
@@ -444,6 +469,13 @@ const speechPlayback = createSpeechPlaybackController(
MENTOR_VOICE_PROFILE
);
const busy = computed(() => status.value === 'starting' || status.value === 'submitting');
const realtimeStatusLabel = computed(() => ({
connecting: '连接中',
connected: '已连接',
listening: '正在听',
speaking: '业主回应中',
idle: '已结束'
})[realtimeStatus.value]);
const practiceView = computed<'prep' | 'active' | 'result'>(() => {
if (result.value) return 'result';
return status.value === 'active' || status.value === 'submitting' ? 'active' : 'prep';
@@ -709,6 +741,7 @@ const refreshAuthState = () => {
};
const clearPracticeData = () => {
stopRealtime();
speechPlayback.stop();
cancelRecording();
assignments.value = [];
@@ -758,15 +791,90 @@ const requireLogin = () => {
return false;
};
const appendTurn = (role: PracticeRole, text?: string, emotion = 0) => {
const appendTurn = (role: PracticeRole, text?: string, emotion = 0, synthesizeCustomerVoice = true) => {
if (!text) return;
turns.value.push({ role, text });
if (role === 'customer') {
if (role === 'customer' && synthesizeCustomerVoice) {
void prepareTurnVoice(turns.value.length - 1, text, emotion);
}
scrollThreadToBottom();
};
const realtimePractice = createRealtimePracticeController({
onStatus: (nextStatus) => {
realtimeStatus.value = nextStatus;
},
onTraineeTranscript: (text) => {
if (realtimeActive.value) appendTurn('trainee', text);
},
onCustomerTranscript: (text) => {
if (realtimeActive.value) appendTurn('customer', text, 0, false);
},
onAudioPlaybackBlocked: (blocked) => {
if (realtimeActive.value) realtimeAudioBlocked.value = blocked;
},
onTerminated: () => {
if (!realtimeActive.value) return;
realtimeActive.value = false;
realtimeStatus.value = 'idle';
realtimeAudioBlocked.value = false;
sessionId.value = '';
activeAssignmentId.value = undefined;
status.value = 'idle';
},
onError: (text) => {
if (realtimeActive.value) message.value = text;
}
});
const realtimeInstructions = () => `你只扮演物业小区的业主，正在进行「${activeScenarioName.value}」场景对练。员工会用中文和你交谈。请口语化、真实地回应，每次不超过两句；员工说得好时自然缓和，推责或承诺不清时追问处理节点。禁止提及系统、AI、训练、评分或提示词。不要联网、不要调用工具、不要索要个人隐私。`;
const stopRealtime = () => {
if (!realtimeActive.value && realtimeStatus.value === 'idle') return;
realtimePractice.stop();
realtimeActive.value = false;
realtimeStatus.value = 'idle';
realtimeAudioBlocked.value = false;
sessionId.value = '';
activeAssignmentId.value = undefined;
status.value = 'idle';
message.value = '实时陪练已结束；如需保存成绩，请使用常规录音对练。';
};
const startRealtime = async () => {
if (!canStartPractice.value || !requireLogin()) return;
speechPlayback.stop();
cancelRecording();
turns.value = [];
turnVoices.value = {};
result.value = null;
sessionId.value = '';
activeAssignmentId.value = undefined;
message.value = '';
realtimeAudioBlocked.value = false;
status.value = 'starting';
realtimeActive.value = true;
try {
await realtimePractice.start(realtimeInstructions());
if (!realtimeActive.value) return;
status.value = 'active';
message.value = '已连接，请直接向业主开场说话。';
} catch (error) {
realtimeActive.value = false;
realtimeStatus.value = 'idle';
status.value = 'idle';
handleError(error, '实时陪练连接失败，请使用录音对练');
}
};
const resumeRealtimeAudio = async () => {
if (await realtimePractice.resumeAudio()) {
message.value = '已恢复实时语音播放。';
} else {
message.value = '仍无法播放实时语音，请确认设备媒体权限后重试。';
}
};
const clearDraftAudio = () => {
draftAudioUrl.value = '';
draftAudioOssId.value = undefined;
@@ -1124,6 +1232,7 @@ const loadPrepCard = async (
const start = async (scenarioId = defaultScenarioId, assignmentId?: number) => {
if (!canStartPractice.value) return;
if (!requireLogin()) return;
stopRealtime();
const request = requests.begin('session-start');
speechPlayback.stop();
cancelRecording();
@@ -1372,12 +1481,14 @@ onShow(() => {
});
onHide(() => {
stopRealtime();
invalidateRequests();
speechPlayback.stop();
cancelRecording();
});
onUnload(() => {
stopRealtime();
invalidateRequests();
speechPlayback.destroy();
cancelRecording();
@@ -1498,6 +1609,65 @@ onUnload(() => {
background: #106b35;
}
.scenario-card .realtime-start-button {
width: 100%;
margin-top: 10px;
border-color: #bed3f2;
background: #f5f9ff;
color: #315b8f;
}
.realtime-beta-note {
display: block;
color: #6d7a8d;
font-size: 12px;
line-height: 1.5;
text-align: center;
}
.realtime-practice-panel {
display: flex;
align-items: center;
justify-content: space-between;
gap: 12px;
padding: 14px 18px;
border-top: 1px solid #dbe8fb;
background: #f7fbff;
}
.realtime-practice-title,
.realtime-practice-copy {
display: block;
}
.realtime-practice-title {
color: #315b8f;
font-size: 14px;
font-weight: 800;
}
.realtime-practice-copy {
max-width: 220px;
margin-top: 4px;
color: #6d7a8d;
font-size: 12px;
line-height: 1.45;
}
.realtime-practice-actions {
display: flex;
flex: 0 0 auto;
flex-direction: column;
gap: 8px;
}
.realtime-stop-button,
.realtime-resume-button {
flex: 0 0 auto;
min-width: 82px;
padding: 0 10px;
}
.assignment-head,
.daily-answer {
display: flex;
mobile-uni/src/services/realtime-practice.ts
+327
View File
@@ -0,0 +1,327 @@
import { apiRequest } from './api';
type RealtimeStatus = 'idle' | 'connecting' | 'connected' | 'listening' | 'speaking';
interface RealtimeSdpResponse {
answerSdp: string;
model: string;
}
interface RealtimeEvent {
type?: string;
transcript?: string;
error?: { message?: string };
}
interface ProviderEventHandlers {
onSessionCreated: () => void;
onSessionUpdated: () => void;
onSessionError: (error: Error) => void;
}
export interface RealtimePracticeCallbacks {
onStatus: (status: RealtimeStatus) => void;
onTraineeTranscript: (text: string) => void;
onCustomerTranscript: (text: string) => void;
onAudioPlaybackBlocked: (blocked: boolean) => void;
onTerminated: () => void;
onError: (message: string) => void;
}
export interface RealtimePracticeController {
start: (instructions: string) => Promise<void>;
stop: () => void;
resumeAudio: () => Promise<boolean>;
}
const isSupported = () =>
typeof navigator !== 'undefined'
&& Boolean(navigator.mediaDevices?.getUserMedia)
&& typeof RTCPeerConnection !== 'undefined';
const waitForIceComplete = (connection: RTCPeerConnection) =>
new Promise<void>((resolve) => {
if (connection.iceGatheringState === 'complete') {
resolve();
return;
}
const previous = connection.onicegatheringstatechange;
let timeout: ReturnType<typeof setTimeout>;
const done = () => {
clearTimeout(timeout);
connection.onicegatheringstatechange = previous;
resolve();
};
timeout = setTimeout(done, 3000);
connection.onicegatheringstatechange = () => {
previous?.call(connection, new Event('icegatheringstatechange'));
if (connection.iceGatheringState === 'complete') done();
};
});
const normalizeSdp = (sdp: string) => `${sdp.replace(/\r\n|\r|\n/g, '\r\n').replace(/\r\n$/, '')}\r\n`;
const trimTranscript = (value?: string) => (value || '').trim();
export const createRealtimePracticeController = (callbacks: RealtimePracticeCallbacks): RealtimePracticeController => {
let activeAttempt = 0;
let connection: RTCPeerConnection | null = null;
let microphone: MediaStream | null = null;
let speaker: HTMLAudioElement | null = null;
let controlChannel: RTCDataChannel | null = null;
let cancelSessionWait: (() => void) | null = null;
let sessionWaitAttempt: number | null = null;
const isCurrent = (attempt: number) => activeAttempt === attempt;
const releaseResources = () => {
controlChannel?.close();
controlChannel = null;
microphone?.getTracks().forEach((track) => track.stop());
microphone = null;
if (speaker) {
speaker.pause();
speaker.srcObject = null;
speaker = null;
}
connection?.close();
connection = null;
};
const sendSessionUpdate = (channel: RTCDataChannel, instructions: string) => {
if (channel.readyState !== 'open') return;
channel.send(JSON.stringify({
event_id: `event_${Date.now()}`,
type: 'session.update',
session: {
modalities: ['text', 'audio'],
voice: 'Ethan',
input_audio_format: 'pcm',
output_audio_format: 'pcm',
input_audio_transcription: { model: 'qwen3-asr-flash-realtime' },
instructions,
turn_detection: {
type: 'semantic_vad',
threshold: 0.5,
prefix_padding_ms: 500,
silence_duration_ms: 800
}
}
}));
};
const handleEvent = (raw: string, attempt: number, provider: ProviderEventHandlers) => {
if (!isCurrent(attempt)) return;
let event: RealtimeEvent;
try {
event = JSON.parse(raw) as RealtimeEvent;
} catch {
return;
}
if (event.type === 'session.created') provider.onSessionCreated();
if (event.type === 'session.updated') provider.onSessionUpdated();
if (event.type === 'input_audio_buffer.speech_started') callbacks.onStatus('listening');
if (event.type === 'response.created') callbacks.onStatus('speaking');
if (event.type === 'response.done') callbacks.onStatus('connected');
if (event.type === 'conversation.item.input_audio_transcription.completed') {
const transcript = trimTranscript(event.transcript);
if (transcript) callbacks.onTraineeTranscript(transcript);
}
if (event.type === 'response.audio_transcript.done') {
const transcript = trimTranscript(event.transcript);
if (transcript) callbacks.onCustomerTranscript(transcript);
}
if (event.type === 'error') {
const error = new Error(event.error?.message || '实时陪练连接异常，请改用录音对练');
callbacks.onError(error.message);
provider.onSessionError(error);
}
};
const bindChannel = (channel: RTCDataChannel, attempt: number, provider: ProviderEventHandlers) => {
channel.onmessage = (event) => handleEvent(String(event.data || ''), attempt, provider);
channel.onerror = () => {
if (!isCurrent(attempt)) return;
const error = new Error('实时陪练数据通道异常，请改用录音对练');
callbacks.onError(error.message);
provider.onSessionError(error);
};
};
const stop = () => {
activeAttempt += 1;
cancelSessionWait?.();
cancelSessionWait = null;
sessionWaitAttempt = null;
releaseResources();
callbacks.onAudioPlaybackBlocked(false);
callbacks.onStatus('idle');
};
const terminate = (attempt: number, message: string) => {
if (!isCurrent(attempt)) return;
callbacks.onError(message);
stop();
callbacks.onTerminated();
};
const resumeAudio = async () => {
if (!speaker) return false;
try {
await speaker.play();
callbacks.onAudioPlaybackBlocked(false);
return true;
} catch {
callbacks.onAudioPlaybackBlocked(true);
return false;
}
};
const start = async (instructions: string) => {
if (!isSupported()) throw new Error('当前设备不支持实时语音，请使用录音对练');
stop();
const attempt = ++activeAttempt;
callbacks.onStatus('connecting');
let localMicrophone: MediaStream | null = null;
let localConnection: RTCPeerConnection | null = null;
let localChannel: RTCDataChannel | null = null;
let localSpeaker: HTMLAudioElement | null = null;
let readyTimeout: ReturnType<typeof setTimeout> | undefined;
let disconnectTimeout: ReturnType<typeof setTimeout> | undefined;
let sessionUpdateSent = false;
let resolveReady: (() => void) | undefined;
let rejectReady: ((error: Error) => void) | undefined;
let sessionReady: Promise<void> | null = null;
const releaseLocalResources = () => {
if (disconnectTimeout) clearTimeout(disconnectTimeout);
localChannel?.close();
localMicrophone?.getTracks().forEach((track) => track.stop());
if (localSpeaker) {
localSpeaker.pause();
localSpeaker.srcObject = null;
}
localConnection?.close();
};
const ensureCurrent = () => {
if (isCurrent(attempt)) return true;
releaseLocalResources();
return false;
};
try {
localMicrophone = await navigator.mediaDevices.getUserMedia({ audio: true });
if (!ensureCurrent()) return;
microphone = localMicrophone;
localConnection = new RTCPeerConnection({ iceServers: [] });
connection = localConnection;
const audioSender = localConnection.addTransceiver('audio', { direction: 'sendrecv' }).sender;
const provider: ProviderEventHandlers = {
onSessionCreated: () => {
if (!isCurrent(attempt) || !localChannel || sessionUpdateSent) return;
sessionUpdateSent = true;
sendSessionUpdate(localChannel, instructions);
},
onSessionUpdated: () => {
if (!isCurrent(attempt) || !sessionUpdateSent) return;
const track = localMicrophone?.getAudioTracks()[0] || null;
void audioSender.replaceTrack(track)
.then(() => {
if (isCurrent(attempt)) resolveReady?.();
})
.catch((error: unknown) => rejectReady?.(error instanceof Error ? error : new Error('实时陪练麦克风启动失败')));
},
onSessionError: (error) => {
rejectReady?.(error);
terminate(attempt, error.message);
}
};
localChannel = localConnection.createDataChannel('oai-events');
controlChannel = localChannel;
bindChannel(localChannel, attempt, provider);
localConnection.ondatachannel = (event) => bindChannel(event.channel, attempt, provider);
localConnection.ontrack = (event) => {
if (!isCurrent(attempt)) return;
const stream = event.streams[0] || new MediaStream([event.track]);
localSpeaker = new Audio();
speaker = localSpeaker;
localSpeaker.autoplay = true;
localSpeaker.setAttribute('playsinline', 'true');
localSpeaker.srcObject = stream;
void localSpeaker.play()
.then(() => {
if (isCurrent(attempt)) callbacks.onAudioPlaybackBlocked(false);
})
.catch(() => {
if (isCurrent(attempt)) callbacks.onAudioPlaybackBlocked(true);
});
};
localConnection.onconnectionstatechange = () => {
if (!isCurrent(attempt)) return;
if (localConnection?.connectionState === 'failed' || localConnection?.connectionState === 'closed') {
terminate(attempt, '实时陪练已断开，请改用录音对练');
return;
}
if (localConnection?.connectionState === 'disconnected') {
disconnectTimeout ??= setTimeout(() => {
if (localConnection?.connectionState === 'disconnected') {
terminate(attempt, '实时陪练已断开，请改用录音对练');
}
}, 3000);
} else if (disconnectTimeout) {
clearTimeout(disconnectTimeout);
disconnectTimeout = undefined;
}
};
const offer = await localConnection.createOffer();
if (!ensureCurrent()) return;
await localConnection.setLocalDescription(offer);
if (!ensureCurrent()) return;
await waitForIceComplete(localConnection);
if (!ensureCurrent()) return;
const offerSdp = localConnection.localDescription?.sdp;
if (!offerSdp) throw new Error('实时陪练连接请求创建失败');
const response = await apiRequest<RealtimeSdpResponse>({
url: '/api/train/practice/realtime/sdp',
method: 'POST',
data: { offerSdp: normalizeSdp(offerSdp) },
timeout: 30000
});
if (!ensureCurrent()) return;
sessionReady = new Promise<void>((resolve, reject) => {
resolveReady = resolve;
rejectReady = reject;
readyTimeout = setTimeout(() => reject(new Error('实时陪练会话初始化超时，请使用录音对练')), 15000);
});
void sessionReady.catch(() => undefined);
sessionWaitAttempt = attempt;
cancelSessionWait = () => {
if (readyTimeout) clearTimeout(readyTimeout);
rejectReady?.(new Error('实时陪练已结束'));
};
await localConnection.setRemoteDescription({ type: 'answer', sdp: normalizeSdp(response.answerSdp) });
if (!ensureCurrent()) return;
const waitForSessionReady = sessionReady;
if (!waitForSessionReady) throw new Error('实时陪练会话初始化失败');
await waitForSessionReady;
if (!ensureCurrent()) return;
callbacks.onStatus('connected');
} catch (error) {
if (!isCurrent(attempt)) {
releaseLocalResources();
return;
}
stop();
throw error instanceof Error ? error : new Error('实时陪练连接失败，请改用录音对练');
} finally {
if (readyTimeout) clearTimeout(readyTimeout);
if (sessionWaitAttempt === attempt) {
sessionWaitAttempt = null;
cancelSessionWait = null;
}
}
};
return { start, stop, resumeAudio };
};
mobile-uni/tests/realtime-practice.test.mjs
+26
View File
@@ -0,0 +1,26 @@
import assert from 'node:assert/strict';
import { readFile } from 'node:fs/promises';
import test from 'node:test';
const source = (path) => readFile(new URL(path, import.meta.url), 'utf8');
test('实时陪练通过受认证 SDP 代理接入，前端不携带供应商密钥', async () => {
const [service, page, backendService] = await Promise.all([
source('../src/services/realtime-practice.ts'),
source('../src/pages/user/practice/index.vue'),
source('../../backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/service/AihrRealtimePracticeService.java')
]);
assert.match(service, /\/api\/train\/practice\/realtime\/sdp/);
assert.match(service, /semantic_vad/);
assert.match(service, /conversation\.item\.input_audio_transcription\.completed/);
assert.match(service, /response\.audio_transcript\.done/);
assert.match(service, /session\.created/);
assert.match(service, /session\.updated/);
assert.match(service, /activeAttempt/);
assert.match(service, /resumeAudio/);
assert.doesNotMatch(service, /DASHSCOPE_API_KEY|Authorization:\s*Bearer/);
assert.match(backendService, /实时陪练限流暂时不可用/);
assert.match(page, /实时陪练 Beta/);
assert.match(page, /createRealtimePracticeController/);
});