feat: support RTF knowledge uploads
This commit is contained in:
+3
-1
@@ -817,7 +817,7 @@ public class AihrSopSeedService {
|
|||||||
}
|
}
|
||||||
String fileName = AihrMultipartFiles.sanitizeFileName(file.getOriginalFilename(), "knowledge.txt");
|
String fileName = AihrMultipartFiles.sanitizeFileName(file.getOriginalFilename(), "knowledge.txt");
|
||||||
if (!supportedFile(fileName)) {
|
if (!supportedFile(fileName)) {
|
||||||
throw new ServiceException("仅支持 txt/md/markdown/pdf/doc/docx/xls/xlsx/ppt/pptx/图片文件");
|
throw new ServiceException("仅支持 txt/md/markdown/rtf/pdf/doc/docx/xls/xlsx/ppt/pptx/图片文件");
|
||||||
}
|
}
|
||||||
if (file.getSize() > 100 * 1024 * 1024) {
|
if (file.getSize() > 100 * 1024 * 1024) {
|
||||||
throw new ServiceException("文件不能超过 100MB");
|
throw new ServiceException("文件不能超过 100MB");
|
||||||
@@ -2588,6 +2588,7 @@ public class AihrSopSeedService {
|
|||||||
case "xls", "xlsx" -> "Excel";
|
case "xls", "xlsx" -> "Excel";
|
||||||
case "ppt", "pptx" -> "PPT";
|
case "ppt", "pptx" -> "PPT";
|
||||||
case "pdf" -> "PDF";
|
case "pdf" -> "PDF";
|
||||||
|
case "rtf" -> "RTF";
|
||||||
case "txt", "md", "markdown" -> "Text";
|
case "txt", "md", "markdown" -> "Text";
|
||||||
case "jpg", "jpeg", "png", "gif", "webp", "bmp" -> "图片";
|
case "jpg", "jpeg", "png", "gif", "webp", "bmp" -> "图片";
|
||||||
case "mp4", "mov", "avi", "mkv", "webm", "m4v" -> "视频";
|
case "mp4", "mov", "avi", "mkv", "webm", "m4v" -> "视频";
|
||||||
@@ -4894,6 +4895,7 @@ public class AihrSopSeedService {
|
|||||||
return textFile(lower)
|
return textFile(lower)
|
||||||
|| imageFile(lower)
|
|| imageFile(lower)
|
||||||
|| AihrVideoService.videoFile(lower)
|
|| AihrVideoService.videoFile(lower)
|
||||||
|
|| lower.endsWith(".rtf")
|
||||||
|| lower.endsWith(".pdf")
|
|| lower.endsWith(".pdf")
|
||||||
|| lower.endsWith(".doc")
|
|| lower.endsWith(".doc")
|
||||||
|| lower.endsWith(".docx")
|
|| lower.endsWith(".docx")
|
||||||
|
|||||||
+1
-1
@@ -121,7 +121,7 @@ public class AihrUploadQueueService {
|
|||||||
String fileName = sanitizeFileName(file.getOriginalFilename());
|
String fileName = sanitizeFileName(file.getOriginalFilename());
|
||||||
boolean archive = isZip(fileName);
|
boolean archive = isZip(fileName);
|
||||||
if (!archive && !AihrSopSeedService.supportedFile(fileName)) {
|
if (!archive && !AihrSopSeedService.supportedFile(fileName)) {
|
||||||
throw new ServiceException("仅支持 txt/md/markdown/pdf/doc/docx/xls/xlsx/ppt/pptx/图片/视频/zip文件");
|
throw new ServiceException("仅支持 txt/md/markdown/rtf/pdf/doc/docx/xls/xlsx/ppt/pptx/图片/视频/zip文件");
|
||||||
}
|
}
|
||||||
if (file.getSize() > MAX_FILE_BYTES) {
|
if (file.getSize() > MAX_FILE_BYTES) {
|
||||||
throw new ServiceException(archive
|
throw new ServiceException(archive
|
||||||
|
|||||||
+14
@@ -44,6 +44,20 @@ class TikaKnowledgeDocumentParserTest {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void parsesRichTextFormatDocuments() {
|
||||||
|
KnowledgeDocumentParser parser = new TikaKnowledgeDocumentParser();
|
||||||
|
byte[] rtf = ("{\\rtf1 \\ansi \\deff0 {\\fonttbl {\\f0 Times New Roman;}}"
|
||||||
|
+ "\\f0 \\fs24 Interview notes: resident service experience.\\par}")
|
||||||
|
.getBytes(StandardCharsets.US_ASCII);
|
||||||
|
|
||||||
|
ParsedDocument document = parser.parse("interview.rtf", "application/rtf", rtf);
|
||||||
|
|
||||||
|
assertEquals("application/rtf", document.mimeType());
|
||||||
|
assertTrue(document.text().contains("Interview notes"));
|
||||||
|
assertFalse(document.segments().isEmpty());
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void rejectsInvalidChunkSettings() {
|
void rejectsInvalidChunkSettings() {
|
||||||
ParsedDocument document = new ParsedDocument("有效内容", "text/plain", java.util.Map.of());
|
ParsedDocument document = new ParsedDocument("有效内容", "text/plain", java.util.Map.of());
|
||||||
|
|||||||
+19
@@ -44,6 +44,25 @@ class AihrUploadQueueServiceTest {
|
|||||||
assertEquals(500L * 1024 * 1024, field.getLong(null));
|
assertEquals(500L * 1024 * 1024, field.getLong(null));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void supportsRtfAtTopLevelAndInsideZip() throws Exception {
|
||||||
|
assertTrue(AihrSopSeedService.supportedFile("interview.rtf"));
|
||||||
|
|
||||||
|
Path staging = Files.createDirectories(tempDir.resolve("rtf-staging"));
|
||||||
|
Path archive = tempDir.resolve("rtf.zip");
|
||||||
|
writeZip(archive, java.nio.charset.StandardCharsets.UTF_8, "notes/interview.rtf");
|
||||||
|
JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class);
|
||||||
|
when(jdbcTemplate.update(anyString(), any(Object[].class))).thenReturn(1);
|
||||||
|
ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
|
||||||
|
try {
|
||||||
|
AihrUploadQueueService service = new AihrUploadQueueService(null, null, jdbcTemplate, executor, staging.toString());
|
||||||
|
|
||||||
|
assertEquals(1, expandArchive(service, itemRow(archive)));
|
||||||
|
} finally {
|
||||||
|
executor.shutdownNow();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void expandsLegacyGbkZipAtomicallyAndKeepsDuplicateEntries() throws Exception {
|
void expandsLegacyGbkZipAtomicallyAndKeepsDuplicateEntries() throws Exception {
|
||||||
Path staging = Files.createDirectories(tempDir.resolve("staging"));
|
Path staging = Files.createDirectories(tempDir.resolve("staging"));
|
||||||
|
|||||||
@@ -135,7 +135,7 @@ SOP 文档上传第三片已经落最小后端边界:
|
|||||||
| 解析状态聚合 | `GET /api/knowledge/processing/overview` | 聚合 `aihr_knowledge_attach.status`、fragment 数、embedding 数、`sys_oss.ext1.fileSize`,生成资料处理页指标、分类、任务、链路和事件列表 |
|
| 解析状态聚合 | `GET /api/knowledge/processing/overview` | 聚合 `aihr_knowledge_attach.status`、fragment 数、embedding 数、`sys_oss.ext1.fileSize`,生成资料处理页指标、分类、任务、链路和事件列表 |
|
||||||
| 服务端目录导入(运维) | `POST /api/knowledge/doc/import-local` | JSON `{ directory, category, limit }`;`directory` 只能是 `AIHR_IMPORT_ROOT` / `aihr.import.root` 下的相对目录,默认根目录为 `./.data/import`;逐文件复用上传解析链路,同步执行,仅保留给运维/调试,不在资料处理页暴露 |
|
| 服务端目录导入(运维) | `POST /api/knowledge/doc/import-local` | JSON `{ directory, category, limit }`;`directory` 只能是 `AIHR_IMPORT_ROOT` / `aihr.import.root` 下的相对目录,默认根目录为 `./.data/import`;逐文件复用上传解析链路,同步执行,仅保留给运维/调试,不在资料处理页暴露 |
|
||||||
| 服务端导入任务(运维) | `POST /api/knowledge/doc/import-local-task`、`GET /api/knowledge/doc/import-tasks`、`POST /api/knowledge/doc/import-tasks/{id}/cancel` | 启动后台目录导入并返回任务;任务写入 `aihr_knowledge_import_task`,调用方可查询总数、成功数、失败数、当前文件和进度,运行中任务可取消;仅保留给运维/调试,不在资料处理页暴露 |
|
| 服务端导入任务(运维) | `POST /api/knowledge/doc/import-local-task`、`GET /api/knowledge/doc/import-tasks`、`POST /api/knowledge/doc/import-tasks/{id}/cancel` | 启动后台目录导入并返回任务;任务写入 `aihr_knowledge_import_task`,调用方可查询总数、成功数、失败数、当前文件和进度,运行中任务可取消;仅保留给运维/调试,不在资料处理页暴露 |
|
||||||
| 批量异步上传 | `POST /api/knowledge/doc/upload-async`(multipart `file`+`category`+`batchId`)、`GET /api/knowledge/doc/upload-items?batchId=`、`POST /api/knowledge/doc/upload-items/{id}/retry`、`POST /api/knowledge/doc/processing-tasks/{attachmentId}/retry` | 上传只做暂存(`aihr.upload.staging`,默认 `./.data/staging`)+ 写入 `aihr_knowledge_upload_item`(0待处理/1处理中/2完成/3失败)即返回;`source_attach_id` 记录从原 OSS 重试的附件。文档、图片、视频和 ZIP 单文件均 ≤500MB,生产 Spring/Undertow multipart 上限为单文件 500MB、请求 520MB,Tika 文档输入流上限同步为 500MiB;worker 限制最多 1000 个 ZIP 子文件、单个受支持子文件 ≤500MB、解压总量 ≤2GB,拒绝嵌套 ZIP/不安全路径并忽略未知格式;ZIP 文件名优先按 UTF-8 解码,旧版中文 Windows ZIP 自动回退 GBK;同名子文件自动追加编号并以单条批量写入入队;只有片段数大于 0 才记为完成并删除暂存文件,零片段或安全媒体错误保持失败态供重试;卡住 30 分钟(视频 120 分钟)由清扫重置,失败暂存文件保留 72h;同步接口 `POST /api/knowledge/doc/upload` 不支持 ZIP,保留给 SOP 页单文件即时预览且仍按 100MB 契约执行 |
|
| 批量异步上传 | `POST /api/knowledge/doc/upload-async`(multipart `file`+`category`+`batchId`)、`GET /api/knowledge/doc/upload-items?batchId=`、`POST /api/knowledge/doc/upload-items/{id}/retry`、`POST /api/knowledge/doc/processing-tasks/{attachmentId}/retry` | 上传只做暂存(`aihr.upload.staging`,默认 `./.data/staging`)+ 写入 `aihr_knowledge_upload_item`(0待处理/1处理中/2完成/3失败)即返回;`source_attach_id` 记录从原 OSS 重试的附件。文档(含 RTF)、图片、视频和 ZIP 单文件均 ≤500MB,生产 Spring/Undertow multipart 上限为单文件 500MB、请求 520MB,Tika 文档输入流上限同步为 500MiB;worker 限制最多 1000 个 ZIP 子文件、单个受支持子文件 ≤500MB、解压总量 ≤2GB,拒绝嵌套 ZIP/不安全路径并忽略未知格式;ZIP 文件名优先按 UTF-8 解码,旧版中文 Windows ZIP 自动回退 GBK;同名子文件自动追加编号并以单条批量写入入队;只有片段数大于 0 才记为完成并删除暂存文件,零片段或安全媒体错误保持失败态供重试;卡住 30 分钟(视频 120 分钟)由清扫重置,失败暂存文件保留 72h;同步接口 `POST /api/knowledge/doc/upload` 不支持 ZIP,保留给 SOP 页单文件即时预览且仍按 100MB 契约执行 |
|
||||||
| 视频解析 | 走批量异步上传,支持 `.mp4/.mov/.avi/.mkv/.webm/.m4v`,单文件 ≤500MB、时长 ≤60 分钟 | 依赖服务器 ffmpeg/ffprobe;抽音轨按 5 分钟分段调 asr 模型转写(`[mm:ss]` 时间戳),按 30s 间隔(≤40 帧)抽关键帧调 vision 模型提取画面文字(相邻重复画面去重);两类文本合并后走归类/切片/向量化;无音轨、ASR 无文本且 vision 无结果时以 `MEDIA_NO_TEXT` 进入可重试失败态,不冒充仍在排队;视频同时只加工 1 个 |
|
| 视频解析 | 走批量异步上传,支持 `.mp4/.mov/.avi/.mkv/.webm/.m4v`,单文件 ≤500MB、时长 ≤60 分钟 | 依赖服务器 ffmpeg/ffprobe;抽音轨按 5 分钟分段调 asr 模型转写(`[mm:ss]` 时间戳),按 30s 间隔(≤40 帧)抽关键帧调 vision 模型提取画面文字(相邻重复画面去重);两类文本合并后走归类/切片/向量化;无音轨、ASR 无文本且 vision 无结果时以 `MEDIA_NO_TEXT` 进入可重试失败态,不冒充仍在排队;视频同时只加工 1 个 |
|
||||||
|
|
||||||
模型能力第二片已经落最小后端边界:
|
模型能力第二片已经落最小后端边界:
|
||||||
|
|||||||
@@ -237,7 +237,7 @@ export function uploadKnowledgeDocAsync(data: FormData): Promise<ApiResult<Uploa
|
|||||||
url: '/api/knowledge/doc/upload-async',
|
url: '/api/knowledge/doc/upload-async',
|
||||||
method: 'post',
|
method: 'post',
|
||||||
data,
|
data,
|
||||||
timeout: 60000,
|
timeout: 600000,
|
||||||
headers: { repeatSubmit: false }
|
headers: { repeatSubmit: false }
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1078,8 +1078,8 @@ import { uploadKnowledgeDocAsync, listUploadItems, retryUploadItem, type UploadI
|
|||||||
import { listKnowledgeSpaces, type KnowledgeSpace } from '@/api/aihr/space';
|
import { listKnowledgeSpaces, type KnowledgeSpace } from '@/api/aihr/space';
|
||||||
import { listPersonalPublishRequests, reviewPersonalPublishRequest, type PersonalPublishRequestRow } from '@/api/aihr/personal';
|
import { listPersonalPublishRequests, reviewPersonalPublishRequest, type PersonalPublishRequestRow } from '@/api/aihr/personal';
|
||||||
|
|
||||||
const acceptTypes = '.txt,.md,.markdown,.pdf,.doc,.docx,.xls,.xlsx,.ppt,.pptx,.jpg,.jpeg,.png,.gif,.webp,.bmp,.mp4,.mov,.avi,.mkv,.webm,.m4v,.zip';
|
const acceptTypes = '.txt,.md,.markdown,.rtf,.pdf,.doc,.docx,.xls,.xlsx,.ppt,.pptx,.jpg,.jpeg,.png,.gif,.webp,.bmp,.mp4,.mov,.avi,.mkv,.webm,.m4v,.zip';
|
||||||
const supportedPattern = /\.(txt|md|markdown|pdf|doc|docx|xls|xlsx|ppt|pptx|jpg|jpeg|png|gif|webp|bmp|mp4|mov|avi|mkv|webm|m4v|zip)$/i;
|
const supportedPattern = /\.(txt|md|markdown|rtf|pdf|doc|docx|xls|xlsx|ppt|pptx|jpg|jpeg|png|gif|webp|bmp|mp4|mov|avi|mkv|webm|m4v|zip)$/i;
|
||||||
const maxFileSize = 500 * 1024 * 1024;
|
const maxFileSize = 500 * 1024 * 1024;
|
||||||
const browserBatchWarnCount = 20;
|
const browserBatchWarnCount = 20;
|
||||||
const browserBatchWarnSize = 200 * 1024 * 1024;
|
const browserBatchWarnSize = 200 * 1024 * 1024;
|
||||||
|
|||||||
@@ -12,7 +12,7 @@
|
|||||||
<el-option v-for="space in manageableSpaces" :key="space.code" :label="space.name" :value="space.code" />
|
<el-option v-for="space in manageableSpaces" :key="space.code" :label="space.name" :value="space.code" />
|
||||||
</el-select>
|
</el-select>
|
||||||
<el-upload
|
<el-upload
|
||||||
accept=".txt,.md,.markdown,.pdf,.doc,.docx,.xls,.xlsx,.ppt,.pptx"
|
accept=".txt,.md,.markdown,.rtf,.pdf,.doc,.docx,.xls,.xlsx,.ppt,.pptx"
|
||||||
:auto-upload="false"
|
:auto-upload="false"
|
||||||
:show-file-list="false"
|
:show-file-list="false"
|
||||||
:on-change="handleDocSelected"
|
:on-change="handleDocSelected"
|
||||||
|
|||||||
Reference in New Issue
Block a user