feat(aihr): support video upload with asr transcript and keyframe ocr
新增 AihrVideoService:ffmpeg 抽音轨按5分钟分段调 asr 转写([mm:ss] 时间戳),30s间隔(≤40帧)抽关键帧调 vision 提取画面文字(重复画面去重), 合并文本复用既有归类/切片/向量化链路。视频只走异步队列(同步接口 拒绝),≤500MB/≤60分钟,同时只加工1个,卡死判定放宽到120分钟;无 asr/vision 结果按「待处理」落库不报错。前端 accept/筛选/文案与 overview 类型徽标同步。端到端验证:30s培训视频语音逐字转写+画面 四要点完整提取,自动归类投诉处理SOP。
This commit is contained in:
+42
-9
@@ -93,13 +93,17 @@ public class AihrSopSeedService {
|
||||
private final ISysOssService ossService;
|
||||
private final String importRootConfig;
|
||||
private final ScheduledExecutorService scheduledExecutorService;
|
||||
private final AihrVideoService videoService;
|
||||
|
||||
public AihrSopSeedService(ObjectMapper objectMapper, JdbcTemplate jdbcTemplate, ISysOssService ossService, @Value("${aihr.import.root:}") String importRootConfig, ScheduledExecutorService scheduledExecutorService) {
|
||||
public AihrSopSeedService(ObjectMapper objectMapper, JdbcTemplate jdbcTemplate, ISysOssService ossService,
|
||||
@Value("${aihr.import.root:}") String importRootConfig,
|
||||
ScheduledExecutorService scheduledExecutorService, AihrVideoService videoService) {
|
||||
this.objectMapper = objectMapper;
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
this.ossService = ossService;
|
||||
this.importRootConfig = importRootConfig;
|
||||
this.scheduledExecutorService = scheduledExecutorService;
|
||||
this.videoService = videoService;
|
||||
}
|
||||
|
||||
public SearchResponse search(SearchRequest request) {
|
||||
@@ -468,8 +472,8 @@ public class AihrSopSeedService {
|
||||
try {
|
||||
content = contentReader.get();
|
||||
if (content.isBlank()) {
|
||||
if (imageFile(fileName)) {
|
||||
// 作战清单 M7：无视觉模型或 OCR 无结果的图片先落库标「待处理」，不算失败
|
||||
if (imageFile(fileName) || AihrVideoService.videoFile(fileName)) {
|
||||
// 作战清单 M7：无视觉模型或 OCR/转写无结果的图片、视频先落库标「待处理」，不算失败
|
||||
return savePendingImage(fileName, category, ossUploader);
|
||||
}
|
||||
throw new ServiceException("文件内容不能为空");
|
||||
@@ -529,19 +533,27 @@ public class AihrSopSeedService {
|
||||
}
|
||||
|
||||
/**
|
||||
* OCR 拿不到文字的图片：只落 OSS + attach 标「等待解析」(status 0)，0 片段返回；
|
||||
* 配置视觉模型后重新上传同名文件即可替换加工。
|
||||
* OCR/转写拿不到文字的图片或视频：只落 OSS + attach 标「等待解析」(status 0)，0 片段返回；
|
||||
* 配置视觉/ASR 模型后重新上传同名文件即可替换加工。
|
||||
*/
|
||||
private UploadResponse savePendingImage(String fileName, String category, Supplier<SysOssVo> ossUploader) {
|
||||
String knowledgeName = isAutoCategory(category) || category == null || category.isBlank() ? "图片素材" : cleanCategory(category);
|
||||
boolean video = AihrVideoService.videoFile(fileName);
|
||||
String defaultName = video ? "培训视频" : "图片素材";
|
||||
String knowledgeName = isAutoCategory(category) || category == null || category.isBlank() ? defaultName : cleanCategory(category);
|
||||
KnowledgeConfig config = knowledgeConfig(knowledgeName);
|
||||
SysOssVo oss = ossUploader.get();
|
||||
String docId = upsertAttach(config.knowledgeId(), oss.getOssId(), fileName);
|
||||
markAttachStatus(config.knowledgeId(), docId, 0, "pending-ocr");
|
||||
String hint = visionRuntime().isPresent()
|
||||
markAttachStatus(config.knowledgeId(), docId, 0, video ? "pending-transcribe" : "pending-ocr");
|
||||
String hint;
|
||||
if (video) {
|
||||
hint = "视频未提取到语音转写或画面文字，已入库标记待处理；确认已启用 asr（语音）或 vision（画面）模型后重新上传即可解析";
|
||||
} else {
|
||||
hint = visionRuntime().isPresent()
|
||||
? "图片 OCR 未提取到文字（当前模型可能不支持图片输入，或图片中无可识别文字），已入库标记待处理"
|
||||
: "未配置可用视觉模型，图片已入库标记待处理；在模型管理启用 vision/多模态 chat 模型后重新上传即可解析";
|
||||
return new UploadResponse(docId, oss.getOssId(), fileName, knowledgeName, 0, hint, List.of("图片", "待OCR"), List.of());
|
||||
}
|
||||
List<String> tags = video ? List.of("视频", "待转写") : List.of("图片", "待OCR");
|
||||
return new UploadResponse(docId, oss.getOssId(), fileName, knowledgeName, 0, hint, tags, List.of());
|
||||
}
|
||||
|
||||
private DocumentInsight documentInsight(String fileName, String content, String requestedCategory) {
|
||||
@@ -1274,6 +1286,8 @@ public class AihrSopSeedService {
|
||||
case "ppt", "pptx" -> "PPT";
|
||||
case "pdf" -> "PDF";
|
||||
case "txt", "md", "markdown" -> "Text";
|
||||
case "jpg", "jpeg", "png", "gif", "webp", "bmp" -> "图片";
|
||||
case "mp4", "mov", "avi", "mkv", "webm", "m4v" -> "视频";
|
||||
default -> type.toUpperCase();
|
||||
};
|
||||
}
|
||||
@@ -1978,6 +1992,10 @@ public class AihrSopSeedService {
|
||||
}
|
||||
|
||||
private String readContent(MultipartFile file, String fileName) {
|
||||
if (AihrVideoService.videoFile(fileName)) {
|
||||
// 视频加工耗时数分钟，只允许走异步队列（暂存路径版本），同步接口直接拒绝
|
||||
throw new ServiceException("视频请使用资料处理中心的批量导入（异步队列）上传");
|
||||
}
|
||||
if (textFile(fileName)) {
|
||||
return normalizeMarkdown(readUtf8(file));
|
||||
}
|
||||
@@ -2007,6 +2025,20 @@ public class AihrSopSeedService {
|
||||
}
|
||||
|
||||
private String readContent(Path file, String fileName) {
|
||||
if (AihrVideoService.videoFile(fileName)) {
|
||||
return videoService.extractText(file, fileName, (bytes, mimeType) -> {
|
||||
Optional<ChatRuntime> runtime = visionRuntime();
|
||||
if (runtime.isEmpty()) {
|
||||
return Optional.empty();
|
||||
}
|
||||
try {
|
||||
return Optional.of(normalizeExtractedText(callVisionOcr(runtime.get(), bytes, mimeType)));
|
||||
} catch (Exception e) {
|
||||
log.warn("video frame ocr failed for {}: {}", fileName, e.getMessage());
|
||||
return Optional.empty();
|
||||
}
|
||||
});
|
||||
}
|
||||
if (textFile(fileName)) {
|
||||
return normalizeMarkdown(readUtf8(file));
|
||||
}
|
||||
@@ -2180,6 +2212,7 @@ public class AihrSopSeedService {
|
||||
String lower = fileName.toLowerCase();
|
||||
return textFile(lower)
|
||||
|| imageFile(lower)
|
||||
|| AihrVideoService.videoFile(lower)
|
||||
|| lower.endsWith(".pdf")
|
||||
|| lower.endsWith(".doc")
|
||||
|| lower.endsWith(".docx")
|
||||
|
||||
+39
-13
@@ -37,11 +37,15 @@ public class AihrUploadQueueService {
|
||||
private static final String TENANT_ID = "000000";
|
||||
private static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("MM-dd HH:mm:ss");
|
||||
private static final long MAX_FILE_BYTES = 100L * 1024 * 1024;
|
||||
/** 视频单独放宽：培训视频普遍大于文档；更大的走服务端目录导入。 */
|
||||
private static final long MAX_VIDEO_BYTES = 500L * 1024 * 1024;
|
||||
private static final int MAX_ERROR_CHARS = 480;
|
||||
/** 加工并发上限：链路含 LLM 归类与 embedding 外呼，并发过高会互相争抢配额并拖慢单文件耗时。 */
|
||||
private static final int WORKER_PERMITS = 2;
|
||||
/** 处理中超过该时长视为孤儿（服务重启/线程异常丢失），由清扫任务重置回待处理。需大于最慢单文件加工时长。 */
|
||||
private static final int STUCK_MINUTES = 30;
|
||||
/** 视频加工链路（ffmpeg+分段ASR+关键帧OCR）耗时更长，卡死判定单独放宽。 */
|
||||
private static final int VIDEO_STUCK_MINUTES = 120;
|
||||
/** 失败条目的暂存文件保留时长，超时由清扫回收磁盘（之后重试会提示重新上传）。 */
|
||||
private static final int STAGING_RETENTION_HOURS = 72;
|
||||
|
||||
@@ -73,10 +77,13 @@ public class AihrUploadQueueService {
|
||||
}
|
||||
String fileName = sanitizeFileName(file.getOriginalFilename());
|
||||
if (!AihrSopSeedService.supportedFile(fileName)) {
|
||||
throw new ServiceException("仅支持 txt/md/markdown/pdf/doc/docx/xls/xlsx/ppt/pptx/图片文件");
|
||||
throw new ServiceException("仅支持 txt/md/markdown/pdf/doc/docx/xls/xlsx/ppt/pptx/图片/视频文件");
|
||||
}
|
||||
if (file.getSize() > MAX_FILE_BYTES) {
|
||||
throw new ServiceException("文件不能超过 100MB");
|
||||
long maxBytes = AihrVideoService.videoFile(fileName) ? MAX_VIDEO_BYTES : MAX_FILE_BYTES;
|
||||
if (file.getSize() > maxBytes) {
|
||||
throw new ServiceException(AihrVideoService.videoFile(fileName)
|
||||
? "视频不能超过 500MB，更大的请放到导入根目录走服务端导入"
|
||||
: "文件不能超过 100MB");
|
||||
}
|
||||
ensureTable();
|
||||
String batch = sanitizeBatchId(batchId);
|
||||
@@ -177,25 +184,30 @@ public class AihrUploadQueueService {
|
||||
|
||||
private Long claimNext() {
|
||||
try {
|
||||
// 同名文件不并发领取：saveDocument 对同 knowledge+fileName 是替换语义，并发会撞 attach 唯一键
|
||||
List<Long> candidates = jdbcTemplate.query("""
|
||||
select i.id from aihr_knowledge_upload_item i
|
||||
// 同名文件不并发领取：saveDocument 对同 knowledge+fileName 是替换语义，并发会撞 attach 唯一键。
|
||||
// 视频最多同时加工 1 个：单个视频要占数分钟的 ffmpeg+ASR+vision，双路并发互相拖慢。
|
||||
boolean videoBusy = hasProcessingVideo();
|
||||
List<QueueCandidate> candidates = jdbcTemplate.query("""
|
||||
select i.id, i.file_name from aihr_knowledge_upload_item i
|
||||
where i.tenant_id = ? and i.status = 0
|
||||
and not exists (
|
||||
select 1 from (select file_name from aihr_knowledge_upload_item where tenant_id = ? and status = 1) p
|
||||
where p.file_name = i.file_name
|
||||
)
|
||||
order by i.id asc
|
||||
limit 5
|
||||
""", (rs, rowNum) -> rs.getLong(1), TENANT_ID, TENANT_ID);
|
||||
for (Long id : candidates) {
|
||||
limit 10
|
||||
""", (rs, rowNum) -> new QueueCandidate(rs.getLong(1), rs.getString(2)), TENANT_ID, TENANT_ID);
|
||||
for (QueueCandidate candidate : candidates) {
|
||||
if (videoBusy && AihrVideoService.videoFile(candidate.fileName())) {
|
||||
continue;
|
||||
}
|
||||
int updated = jdbcTemplate.update("""
|
||||
update aihr_knowledge_upload_item
|
||||
set status = 1, update_time = now()
|
||||
where tenant_id = ? and id = ? and status = 0
|
||||
""", TENANT_ID, id);
|
||||
""", TENANT_ID, candidate.id());
|
||||
if (updated == 1) {
|
||||
return id;
|
||||
return candidate.id();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
@@ -205,6 +217,16 @@ public class AihrUploadQueueService {
|
||||
}
|
||||
}
|
||||
|
||||
private boolean hasProcessingVideo() {
|
||||
List<String> processing = jdbcTemplate.query("""
|
||||
select file_name from aihr_knowledge_upload_item where tenant_id = ? and status = 1
|
||||
""", (rs, rowNum) -> rs.getString(1), TENANT_ID);
|
||||
return processing.stream().anyMatch(AihrVideoService::videoFile);
|
||||
}
|
||||
|
||||
private record QueueCandidate(Long id, String fileName) {
|
||||
}
|
||||
|
||||
private void processItem(Long id) {
|
||||
ItemRow row = jdbcTemplate.query("""
|
||||
select id, file_name, category, staging_path, status
|
||||
@@ -249,8 +271,12 @@ public class AihrUploadQueueService {
|
||||
int reset = jdbcTemplate.update("""
|
||||
update aihr_knowledge_upload_item
|
||||
set status = 0, update_time = now()
|
||||
where tenant_id = ? and status = 1 and update_time < (now() - interval %d minute)
|
||||
""".formatted(STUCK_MINUTES), TENANT_ID);
|
||||
where tenant_id = ? and status = 1
|
||||
and (
|
||||
(update_time < (now() - interval %d minute) and file_name not regexp '\\\\.(mp4|mov|avi|mkv|webm|m4v)$')
|
||||
or update_time < (now() - interval %d minute)
|
||||
)
|
||||
""".formatted(STUCK_MINUTES, VIDEO_STUCK_MINUTES), TENANT_ID);
|
||||
if (reset > 0) {
|
||||
log.info("upload queue reset {} stuck items", reset);
|
||||
}
|
||||
|
||||
+223
@@ -0,0 +1,223 @@
|
||||
package org.dromara.aihr.service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.dromara.common.core.exception.ServiceException;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/**
|
||||
* 视频加工：ffmpeg 抽音轨分段 + ASR 转写（带时间戳），抽关键帧 + 视觉 OCR（PPT 录屏字幕类画面文字）。
|
||||
* 产物是纯文本，供上层复用既有的归类/切片/向量化链路。视觉 OCR 由调用方注入（避免与 SOP 服务循环依赖）。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class AihrVideoService {
|
||||
|
||||
/** 单段音频时长（秒）：SenseVoice 对长音频不友好，分段转写后拼接。 */
|
||||
private static final int AUDIO_SEGMENT_SECONDS = 300;
|
||||
/** 关键帧抽取间隔（秒）与上限：控制 vision 调用成本（G6）。 */
|
||||
private static final int FRAME_INTERVAL_SECONDS = 30;
|
||||
private static final int MAX_FRAMES = 40;
|
||||
private static final int MAX_AUDIO_SEGMENTS = 12;
|
||||
/** 支持的最长视频（秒），超长直接拒绝，引导切分后再上传。 */
|
||||
private static final int MAX_DURATION_SECONDS = 3600;
|
||||
private static final int FFMPEG_TIMEOUT_SECONDS = 300;
|
||||
|
||||
private final AihrSpeechService speechService;
|
||||
|
||||
/** 视觉 OCR 回调：入参 (imageBytes, mimeType)，返回识别文本；不可用/失败返回 empty。 */
|
||||
@FunctionalInterface
|
||||
public interface FrameOcr {
|
||||
Optional<String> ocr(byte[] imageBytes, String mimeType);
|
||||
}
|
||||
|
||||
public static boolean videoFile(String fileName) {
|
||||
String lower = fileName == null ? "" : fileName.toLowerCase(Locale.ROOT);
|
||||
return lower.endsWith(".mp4") || lower.endsWith(".mov") || lower.endsWith(".avi")
|
||||
|| lower.endsWith(".mkv") || lower.endsWith(".webm") || lower.endsWith(".m4v");
|
||||
}
|
||||
|
||||
public boolean ffmpegAvailable() {
|
||||
return commandAvailable("ffmpeg") && commandAvailable("ffprobe");
|
||||
}
|
||||
|
||||
/**
|
||||
* 提取视频文本：音轨转写（[mm:ss] 前缀）+ 关键帧画面文字（[画面 mm:ss] 前缀）。
|
||||
* ASR 未配置且视觉不可用时返回空串，由上层按「待处理」落库（同图片语义）。
|
||||
*/
|
||||
public String extractText(Path video, String fileName, FrameOcr frameOcr) {
|
||||
if (!ffmpegAvailable()) {
|
||||
throw new ServiceException("服务器未安装 ffmpeg/ffprobe，无法解析视频");
|
||||
}
|
||||
long durationSeconds = probeDurationSeconds(video);
|
||||
if (durationSeconds > MAX_DURATION_SECONDS) {
|
||||
throw new ServiceException("视频超过 60 分钟，请切分后再上传");
|
||||
}
|
||||
Path workDir = null;
|
||||
try {
|
||||
workDir = Files.createTempDirectory("aihr-video-");
|
||||
StringBuilder text = new StringBuilder();
|
||||
String transcript = transcribeAudio(video, workDir, durationSeconds);
|
||||
if (!transcript.isBlank()) {
|
||||
text.append("【语音转写】\n").append(transcript).append('\n');
|
||||
}
|
||||
String frameText = ocrKeyFrames(video, workDir, durationSeconds, frameOcr);
|
||||
if (!frameText.isBlank()) {
|
||||
text.append("【画面文字】\n").append(frameText).append('\n');
|
||||
}
|
||||
return text.toString().strip();
|
||||
} catch (ServiceException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
log.warn("video extract failed for {}: {}", fileName, e.getMessage());
|
||||
throw new ServiceException("视频解析失败：" + e.getMessage());
|
||||
} finally {
|
||||
cleanDirQuietly(workDir);
|
||||
}
|
||||
}
|
||||
|
||||
private String transcribeAudio(Path video, Path workDir, long durationSeconds) throws IOException, InterruptedException {
|
||||
if (!speechService.asrConfigured()) {
|
||||
log.info("asr not configured, skip video transcription");
|
||||
return "";
|
||||
}
|
||||
int segments = (int) Math.min(MAX_AUDIO_SEGMENTS, (durationSeconds + AUDIO_SEGMENT_SECONDS - 1) / AUDIO_SEGMENT_SECONDS);
|
||||
StringBuilder transcript = new StringBuilder();
|
||||
for (int i = 0; i < segments; i++) {
|
||||
long offset = (long) i * AUDIO_SEGMENT_SECONDS;
|
||||
Path segment = workDir.resolve("audio-" + i + ".mp3");
|
||||
// 16kHz 单声道 mp3：ASR 友好且体积小
|
||||
int exit = runFfmpeg(List.of(
|
||||
"ffmpeg", "-y", "-nostdin", "-v", "error",
|
||||
"-ss", String.valueOf(offset), "-t", String.valueOf(AUDIO_SEGMENT_SECONDS),
|
||||
"-i", video.toString(),
|
||||
"-vn", "-ac", "1", "-ar", "16000", "-b:a", "48k",
|
||||
segment.toString()
|
||||
));
|
||||
if (exit != 0 || !Files.isRegularFile(segment) || Files.size(segment) < 1024) {
|
||||
// 无音轨或该段为空（比如纯画面视频），跳过不算失败
|
||||
continue;
|
||||
}
|
||||
Optional<String> piece = speechService.transcribe(Files.readAllBytes(segment), "segment-" + i + ".mp3", "audio/mpeg");
|
||||
piece.filter(s -> !s.isBlank())
|
||||
.ifPresent(s -> transcript.append('[').append(formatTime(offset)).append("] ").append(s.strip()).append('\n'));
|
||||
Files.deleteIfExists(segment);
|
||||
}
|
||||
return transcript.toString();
|
||||
}
|
||||
|
||||
private String ocrKeyFrames(Path video, Path workDir, long durationSeconds, FrameOcr frameOcr) throws IOException, InterruptedException {
|
||||
if (frameOcr == null) {
|
||||
return "";
|
||||
}
|
||||
long step = Math.max(FRAME_INTERVAL_SECONDS, durationSeconds / MAX_FRAMES + 1);
|
||||
StringBuilder frameText = new StringBuilder();
|
||||
String lastText = "";
|
||||
for (long offset = 0; offset < Math.max(durationSeconds, 1); offset += step) {
|
||||
Path frame = workDir.resolve("frame-" + offset + ".jpg");
|
||||
int exit = runFfmpeg(List.of(
|
||||
"ffmpeg", "-y", "-nostdin", "-v", "error",
|
||||
"-ss", String.valueOf(offset), "-i", video.toString(),
|
||||
"-frames:v", "1", "-q:v", "4", "-vf", "scale='min(1280,iw)':-2",
|
||||
frame.toString()
|
||||
));
|
||||
if (exit != 0 || !Files.isRegularFile(frame)) {
|
||||
continue;
|
||||
}
|
||||
String text = frameOcr.ocr(Files.readAllBytes(frame), "image/jpeg").orElse("").strip();
|
||||
Files.deleteIfExists(frame);
|
||||
// 相邻帧文字几乎相同（静止幻灯片）时去重，避免片段灌水
|
||||
if (!text.isBlank() && !similarText(text, lastText)) {
|
||||
frameText.append("[画面 ").append(formatTime(offset)).append("] ").append(text).append('\n');
|
||||
lastText = text;
|
||||
}
|
||||
}
|
||||
return frameText.toString();
|
||||
}
|
||||
|
||||
private long probeDurationSeconds(Path video) {
|
||||
try {
|
||||
Process process = new ProcessBuilder(
|
||||
"ffprobe", "-v", "error", "-show_entries", "format=duration",
|
||||
"-of", "default=noprint_wrappers=1:nokey=1", video.toString()
|
||||
).redirectErrorStream(true).start();
|
||||
String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8).strip();
|
||||
if (!process.waitFor(30, TimeUnit.SECONDS)) {
|
||||
process.destroyForcibly();
|
||||
throw new ServiceException("视频时长探测超时");
|
||||
}
|
||||
return (long) Double.parseDouble(output.lines().findFirst().orElse("0"));
|
||||
} catch (ServiceException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
throw new ServiceException("视频文件无法识别：" + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private int runFfmpeg(List<String> command) throws IOException, InterruptedException {
|
||||
Process process = new ProcessBuilder(command).redirectErrorStream(true).start();
|
||||
// 消费输出防止缓冲区塞满导致进程挂起
|
||||
byte[] discarded = process.getInputStream().readAllBytes();
|
||||
if (!process.waitFor(FFMPEG_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
|
||||
process.destroyForcibly();
|
||||
throw new ServiceException("ffmpeg 处理超时");
|
||||
}
|
||||
int exit = process.exitValue();
|
||||
if (exit != 0) {
|
||||
log.debug("ffmpeg exit {}: {}", exit, new String(discarded, StandardCharsets.UTF_8));
|
||||
}
|
||||
return exit;
|
||||
}
|
||||
|
||||
private static boolean commandAvailable(String command) {
|
||||
try {
|
||||
Process process = new ProcessBuilder(command, "-version").redirectErrorStream(true).start();
|
||||
process.getInputStream().readAllBytes();
|
||||
return process.waitFor(10, TimeUnit.SECONDS) && process.exitValue() == 0;
|
||||
} catch (Exception e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** 简易相似判定：前 80 字符相同视为同一画面（幻灯片停留场景）。 */
|
||||
private static boolean similarText(String a, String b) {
|
||||
if (a.isBlank() || b.isBlank()) {
|
||||
return false;
|
||||
}
|
||||
int len = Math.min(80, Math.min(a.length(), b.length()));
|
||||
return a.regionMatches(0, b, 0, len);
|
||||
}
|
||||
|
||||
private static String formatTime(long seconds) {
|
||||
return "%02d:%02d".formatted(seconds / 60, seconds % 60);
|
||||
}
|
||||
|
||||
private static void cleanDirQuietly(Path dir) {
|
||||
if (dir == null) {
|
||||
return;
|
||||
}
|
||||
try (Stream<Path> stream = Files.walk(dir)) {
|
||||
stream.sorted(Comparator.reverseOrder()).forEach(path -> {
|
||||
try {
|
||||
Files.deleteIfExists(path);
|
||||
} catch (IOException ignored) {
|
||||
// 临时目录清理失败不影响业务
|
||||
}
|
||||
});
|
||||
} catch (IOException ignored) {
|
||||
// 同上
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -55,7 +55,8 @@ SOP 文档上传第三片已经落最小后端边界：
|
||||
| 解析状态聚合 | `GET /api/knowledge/processing/overview` | 聚合 `aihr_knowledge_attach.status`、fragment 数、embedding 数、`sys_oss.ext1.fileSize`，生成资料处理页指标、分类、任务、链路和事件列表 |
|
||||
| 服务端目录导入 | `POST /api/knowledge/doc/import-local` | JSON `{ directory, category, limit }`；`directory` 只能是 `AIHR_IMPORT_ROOT` / `aihr.import.root` 下的相对目录，默认根目录为 `./.data/import`；逐文件复用上传解析链路，同步执行，保留给小批量/调试 |
|
||||
| 服务端导入任务 | `POST /api/knowledge/doc/import-local-task`、`GET /api/knowledge/doc/import-tasks`、`POST /api/knowledge/doc/import-tasks/{id}/cancel` | 启动后台目录导入并返回任务；任务写入 `aihr_knowledge_import_task`，页面轮询查看总数、成功数、失败数、当前文件和进度；运行中任务可取消；重试当前按同目录重新启动一轮 |
|
||||
| 批量异步上传 | `POST /api/knowledge/doc/upload-async`（multipart `file`+`category`+`batchId`）、`GET /api/knowledge/doc/upload-items?batchId=`、`POST /api/knowledge/doc/upload-items/{id}/retry` | 上传只做暂存（`aihr.upload.staging`，默认 `./.data/staging`）+ 写入 `aihr_knowledge_upload_item`（0待处理/1处理中/2完成/3失败）即返回；后台 worker 并发 2 复用同一解析/归类/向量化链路，同名文件串行防撞唯一键；完成/失败状态 CAS 写入，卡住 30 分钟由清扫重置，失败暂存文件保留 72h 供重试；同步接口 `POST /api/knowledge/doc/upload` 保留给 SOP 页单文件即时预览 |
|
||||
| 批量异步上传 | `POST /api/knowledge/doc/upload-async`（multipart `file`+`category`+`batchId`）、`GET /api/knowledge/doc/upload-items?batchId=`、`POST /api/knowledge/doc/upload-items/{id}/retry` | 上传只做暂存（`aihr.upload.staging`，默认 `./.data/staging`）+ 写入 `aihr_knowledge_upload_item`（0待处理/1处理中/2完成/3失败）即返回；后台 worker 并发 2 复用同一解析/归类/向量化链路，同名文件串行防撞唯一键；完成/失败状态 CAS 写入，卡住 30 分钟（视频 120 分钟）由清扫重置，失败暂存文件保留 72h 供重试；同步接口 `POST /api/knowledge/doc/upload` 保留给 SOP 页单文件即时预览 |
|
||||
| 视频解析 | 走批量异步上传，支持 `.mp4/.mov/.avi/.mkv/.webm/.m4v`，单文件 ≤500MB、时长 ≤60 分钟 | 依赖服务器 ffmpeg/ffprobe；抽音轨按 5 分钟分段调 asr 模型转写（`[mm:ss]` 时间戳），按 30s 间隔（≤40 帧）抽关键帧调 vision 模型提取画面文字（相邻重复画面去重）；两类文本合并后走归类/切片/向量化；无 asr 且无 vision 结果时按「待处理」落库不报错；视频同时只加工 1 个 |
|
||||
|
||||
模型能力第二片已经落最小后端边界：
|
||||
|
||||
|
||||
@@ -68,6 +68,7 @@ AIHR_SMS_DEV_FIXED_CODE=123456
|
||||
- `aihr_practice_mysql8.sql` 已导入；移动端员工训练记录落 `aihr_practice_session`，用于训练历史、主管待复盘列表和能力画像聚合
|
||||
- `aihr_org_snapshot_mysql8.sql` 已纳入 reset 脚本；组织人员静态快照（2 个住宅项目 22 人，项目经理/主管/一线三层），支撑按项目看人数演示叙事
|
||||
- SOP 知识库支持 `.txt/.md/.markdown/.pdf/.doc/.docx/.xls/.xlsx/.ppt/.pptx` 上传到 MinIO 后解析入库，接口为 `POST /api/knowledge/doc/upload`，单文件上限 100MB；管理端上传请求单独放宽到 180s，PDF 解析/归类/向量化较慢时不要改全局 axios 超时
|
||||
- 视频（`.mp4/.mov/.avi/.mkv/.webm/.m4v`，≤500MB、≤60 分钟）只走资料处理中心批量导入（异步队列）：ffmpeg 抽音轨分段调 asr 转写 + 抽关键帧调 vision 提取画面文字，合并后归类切片入库，片段带 `[mm:ss]` 时间戳；**依赖服务器安装 ffmpeg/ffprobe**（macOS `brew install ffmpeg`，Linux `apt install ffmpeg`）
|
||||
- 服务端目录导入接口为 `POST /api/knowledge/doc/import-local-task`，只读取导入根目录下的相对目录，后台逐文件复用同一上传解析链路；`POST /api/knowledge/doc/import-tasks/{id}/cancel` 可取消运行中任务；`POST /api/knowledge/doc/import-local` 保留为同步调试接口。
|
||||
- 启用 `aihr_model_config.category='vector'` 的模型配置后，上传会同步写入片段 embedding，并尽力 upsert 到 Qdrant；模型配置页可查看 Qdrant 维度、点数和片段向量数，并在维度不一致时重建索引；未配置或 Qdrant 不可用时只走 MySQL Fulltext/seed fallback，不影响检索。
|
||||
- 后端 `ruoyi-admin` 已完成 Maven 打包并启动在 `8080`
|
||||
|
||||
@@ -290,10 +290,12 @@ import {
|
||||
} from '@/api/aihr/processing';
|
||||
import { uploadKnowledgeDocAsync, listUploadItems, retryUploadItem, type UploadItem } from '@/api/aihr/sop';
|
||||
|
||||
const acceptTypes = '.txt,.md,.markdown,.pdf,.doc,.docx,.xls,.xlsx,.ppt,.pptx,.jpg,.jpeg,.png,.gif,.webp,.bmp';
|
||||
const acceptTypes = '.txt,.md,.markdown,.pdf,.doc,.docx,.xls,.xlsx,.ppt,.pptx,.jpg,.jpeg,.png,.gif,.webp,.bmp,.mp4,.mov,.avi,.mkv,.webm,.m4v';
|
||||
const autoCategory = '__auto__';
|
||||
const supportedPattern = /\.(txt|md|markdown|pdf|doc|docx|xls|xlsx|ppt|pptx|jpg|jpeg|png|gif|webp|bmp)$/i;
|
||||
const supportedPattern = /\.(txt|md|markdown|pdf|doc|docx|xls|xlsx|ppt|pptx|jpg|jpeg|png|gif|webp|bmp|mp4|mov|avi|mkv|webm|m4v)$/i;
|
||||
const videoPattern = /\.(mp4|mov|avi|mkv|webm|m4v)$/i;
|
||||
const maxFileSize = 100 * 1024 * 1024;
|
||||
const maxVideoSize = 500 * 1024 * 1024;
|
||||
const browserBatchWarnCount = 20;
|
||||
const browserBatchWarnSize = 200 * 1024 * 1024;
|
||||
|
||||
@@ -319,7 +321,7 @@ let importPollTimer: ReturnType<typeof window.setTimeout> | undefined;
|
||||
let batchPollTimer: ReturnType<typeof window.setTimeout> | undefined;
|
||||
|
||||
const statusTabs = ['全部', '等待解析', '解析中', '已完成', '失败'];
|
||||
const fileTypes = ['全部', 'Word', 'PDF', 'PPT', 'Excel', '图片'];
|
||||
const fileTypes = ['全部', 'Word', 'PDF', 'PPT', 'Excel', '图片', '视频'];
|
||||
|
||||
const seedOverview: ProcessingOverview = {
|
||||
metrics: [
|
||||
@@ -488,13 +490,15 @@ async function handlePickedFiles(event: Event) {
|
||||
}
|
||||
|
||||
async function uploadFiles(files: File[]) {
|
||||
const readyFiles = files.filter((file) => supportedPattern.test(file.name) && file.size > 0 && file.size <= maxFileSize);
|
||||
const readyFiles = files.filter(
|
||||
(file) => supportedPattern.test(file.name) && file.size > 0 && file.size <= (videoPattern.test(file.name) ? maxVideoSize : maxFileSize)
|
||||
);
|
||||
if (readyFiles.length === 0) {
|
||||
ElMessage.warning('仅支持非空且 100MB 内的 txt/md/pdf/Word/Excel/PPT/图片文件');
|
||||
ElMessage.warning('仅支持非空的 txt/md/pdf/Word/Excel/PPT/图片（100MB内）或视频（500MB内）文件');
|
||||
return;
|
||||
}
|
||||
if (readyFiles.length < files.length) {
|
||||
ElMessage.warning(`已跳过 ${files.length - readyFiles.length} 个不支持、空文件或超过 100MB 的文件`);
|
||||
ElMessage.warning(`已跳过 ${files.length - readyFiles.length} 个不支持、空文件或超限文件（文档/图片 100MB、视频 500MB）`);
|
||||
}
|
||||
|
||||
const totalSize = readyFiles.reduce((sum, file) => sum + file.size, 0);
|
||||
@@ -772,6 +776,7 @@ function fileShort(type: string) {
|
||||
if (type === 'Excel') return 'X';
|
||||
if (type === 'PPT') return 'P';
|
||||
if (type === '图片') return '图';
|
||||
if (type === '视频') return '视';
|
||||
return 'T';
|
||||
}
|
||||
|
||||
@@ -781,6 +786,7 @@ function fileBadgeClass(type: string) {
|
||||
if (type === 'Excel') return 'excel';
|
||||
if (type === 'PPT') return 'ppt';
|
||||
if (type === '图片') return 'image';
|
||||
if (type === '视频') return 'ppt';
|
||||
return 'text';
|
||||
}
|
||||
</script>
|
||||
|
||||
Reference in New Issue
Block a user