feat: support RTF knowledge uploads
This commit is contained in:
key
+3
-1
@@ -817,7 +817,7 @@ public class AihrSopSeedService {
|
||||
}
|
||||
String fileName = AihrMultipartFiles.sanitizeFileName(file.getOriginalFilename(), "knowledge.txt");
|
||||
if (!supportedFile(fileName)) {
|
||||
throw new ServiceException("仅支持 txt/md/markdown/pdf/doc/docx/xls/xlsx/ppt/pptx/图片文件");
|
||||
throw new ServiceException("仅支持 txt/md/markdown/rtf/pdf/doc/docx/xls/xlsx/ppt/pptx/图片文件");
|
||||
}
|
||||
if (file.getSize() > 100 * 1024 * 1024) {
|
||||
throw new ServiceException("文件不能超过 100MB");
|
||||
@@ -2588,6 +2588,7 @@ public class AihrSopSeedService {
|
||||
case "xls", "xlsx" -> "Excel";
|
||||
case "ppt", "pptx" -> "PPT";
|
||||
case "pdf" -> "PDF";
|
||||
case "rtf" -> "RTF";
|
||||
case "txt", "md", "markdown" -> "Text";
|
||||
case "jpg", "jpeg", "png", "gif", "webp", "bmp" -> "图片";
|
||||
case "mp4", "mov", "avi", "mkv", "webm", "m4v" -> "视频";
|
||||
@@ -4894,6 +4895,7 @@ public class AihrSopSeedService {
|
||||
return textFile(lower)
|
||||
|| imageFile(lower)
|
||||
|| AihrVideoService.videoFile(lower)
|
||||
|| lower.endsWith(".rtf")
|
||||
|| lower.endsWith(".pdf")
|
||||
|| lower.endsWith(".doc")
|
||||
|| lower.endsWith(".docx")
|
||||
|
||||
+1
-1
@@ -121,7 +121,7 @@ public class AihrUploadQueueService {
|
||||
String fileName = sanitizeFileName(file.getOriginalFilename());
|
||||
boolean archive = isZip(fileName);
|
||||
if (!archive && !AihrSopSeedService.supportedFile(fileName)) {
|
||||
throw new ServiceException("仅支持 txt/md/markdown/pdf/doc/docx/xls/xlsx/ppt/pptx/图片/视频/zip文件");
|
||||
throw new ServiceException("仅支持 txt/md/markdown/rtf/pdf/doc/docx/xls/xlsx/ppt/pptx/图片/视频/zip文件");
|
||||
}
|
||||
if (file.getSize() > MAX_FILE_BYTES) {
|
||||
throw new ServiceException(archive
|
||||
|
||||
+14
@@ -44,6 +44,20 @@ class TikaKnowledgeDocumentParserTest {
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void parsesRichTextFormatDocuments() {
|
||||
KnowledgeDocumentParser parser = new TikaKnowledgeDocumentParser();
|
||||
byte[] rtf = ("{\\rtf1 \\ansi \\deff0 {\\fonttbl {\\f0 Times New Roman;}}"
|
||||
+ "\\f0 \\fs24 Interview notes: resident service experience.\\par}")
|
||||
.getBytes(StandardCharsets.US_ASCII);
|
||||
|
||||
ParsedDocument document = parser.parse("interview.rtf", "application/rtf", rtf);
|
||||
|
||||
assertEquals("application/rtf", document.mimeType());
|
||||
assertTrue(document.text().contains("Interview notes"));
|
||||
assertFalse(document.segments().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsInvalidChunkSettings() {
|
||||
ParsedDocument document = new ParsedDocument("有效内容", "text/plain", java.util.Map.of());
|
||||
|
||||
+19
@@ -44,6 +44,25 @@ class AihrUploadQueueServiceTest {
|
||||
assertEquals(500L * 1024 * 1024, field.getLong(null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void supportsRtfAtTopLevelAndInsideZip() throws Exception {
|
||||
assertTrue(AihrSopSeedService.supportedFile("interview.rtf"));
|
||||
|
||||
Path staging = Files.createDirectories(tempDir.resolve("rtf-staging"));
|
||||
Path archive = tempDir.resolve("rtf.zip");
|
||||
writeZip(archive, java.nio.charset.StandardCharsets.UTF_8, "notes/interview.rtf");
|
||||
JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class);
|
||||
when(jdbcTemplate.update(anyString(), any(Object[].class))).thenReturn(1);
|
||||
ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
|
||||
try {
|
||||
AihrUploadQueueService service = new AihrUploadQueueService(null, null, jdbcTemplate, executor, staging.toString());
|
||||
|
||||
assertEquals(1, expandArchive(service, itemRow(archive)));
|
||||
} finally {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void expandsLegacyGbkZipAtomicallyAndKeepsDuplicateEntries() throws Exception {
|
||||
Path staging = Files.createDirectories(tempDir.resolve("staging"));
|
||||
|
||||
@@ -135,7 +135,7 @@ SOP 文档上传第三片已经落最小后端边界：
|
||||
| 解析状态聚合 | `GET /api/knowledge/processing/overview` | 聚合 `aihr_knowledge_attach.status`、fragment 数、embedding 数、`sys_oss.ext1.fileSize`，生成资料处理页指标、分类、任务、链路和事件列表 |
|
||||
| 服务端目录导入（运维） | `POST /api/knowledge/doc/import-local` | JSON `{ directory, category, limit }`；`directory` 只能是 `AIHR_IMPORT_ROOT` / `aihr.import.root` 下的相对目录，默认根目录为 `./.data/import`；逐文件复用上传解析链路，同步执行，仅保留给运维/调试，不在资料处理页暴露 |
|
||||
| 服务端导入任务（运维） | `POST /api/knowledge/doc/import-local-task`、`GET /api/knowledge/doc/import-tasks`、`POST /api/knowledge/doc/import-tasks/{id}/cancel` | 启动后台目录导入并返回任务；任务写入 `aihr_knowledge_import_task`，调用方可查询总数、成功数、失败数、当前文件和进度，运行中任务可取消；仅保留给运维/调试，不在资料处理页暴露 |
|
||||
| 批量异步上传 | `POST /api/knowledge/doc/upload-async`（multipart `file`+`category`+`batchId`）、`GET /api/knowledge/doc/upload-items?batchId=`、`POST /api/knowledge/doc/upload-items/{id}/retry`、`POST /api/knowledge/doc/processing-tasks/{attachmentId}/retry` | 上传只做暂存（`aihr.upload.staging`，默认 `./.data/staging`）+ 写入 `aihr_knowledge_upload_item`（0待处理/1处理中/2完成/3失败）即返回；`source_attach_id` 记录从原 OSS 重试的附件。文档、图片、视频和 ZIP 单文件均 ≤500MB，生产 Spring/Undertow multipart 上限为单文件 500MB、请求 520MB，Tika 文档输入流上限同步为 500MiB；worker 限制最多 1000 个 ZIP 子文件、单个受支持子文件 ≤500MB、解压总量 ≤2GB，拒绝嵌套 ZIP/不安全路径并忽略未知格式；ZIP 文件名优先按 UTF-8 解码，旧版中文 Windows ZIP 自动回退 GBK；同名子文件自动追加编号并以单条批量写入入队；只有片段数大于 0 才记为完成并删除暂存文件，零片段或安全媒体错误保持失败态供重试；卡住 30 分钟（视频 120 分钟）由清扫重置，失败暂存文件保留 72h；同步接口 `POST /api/knowledge/doc/upload` 不支持 ZIP，保留给 SOP 页单文件即时预览且仍按 100MB 契约执行 |
|
||||
| 批量异步上传 | `POST /api/knowledge/doc/upload-async`（multipart `file`+`category`+`batchId`）、`GET /api/knowledge/doc/upload-items?batchId=`、`POST /api/knowledge/doc/upload-items/{id}/retry`、`POST /api/knowledge/doc/processing-tasks/{attachmentId}/retry` | 上传只做暂存（`aihr.upload.staging`，默认 `./.data/staging`）+ 写入 `aihr_knowledge_upload_item`（0待处理/1处理中/2完成/3失败）即返回；`source_attach_id` 记录从原 OSS 重试的附件。文档（含 RTF）、图片、视频和 ZIP 单文件均 ≤500MB，生产 Spring/Undertow multipart 上限为单文件 500MB、请求 520MB，Tika 文档输入流上限同步为 500MiB；worker 限制最多 1000 个 ZIP 子文件、单个受支持子文件 ≤500MB、解压总量 ≤2GB，拒绝嵌套 ZIP/不安全路径并忽略未知格式；ZIP 文件名优先按 UTF-8 解码，旧版中文 Windows ZIP 自动回退 GBK；同名子文件自动追加编号并以单条批量写入入队；只有片段数大于 0 才记为完成并删除暂存文件，零片段或安全媒体错误保持失败态供重试；卡住 30 分钟（视频 120 分钟）由清扫重置，失败暂存文件保留 72h；同步接口 `POST /api/knowledge/doc/upload` 不支持 ZIP，保留给 SOP 页单文件即时预览且仍按 100MB 契约执行 |
|
||||
| 视频解析 | 走批量异步上传，支持 `.mp4/.mov/.avi/.mkv/.webm/.m4v`，单文件 ≤500MB、时长 ≤60 分钟 | 依赖服务器 ffmpeg/ffprobe；抽音轨按 5 分钟分段调 asr 模型转写（`[mm:ss]` 时间戳），按 30s 间隔（≤40 帧）抽关键帧调 vision 模型提取画面文字（相邻重复画面去重）；两类文本合并后走归类/切片/向量化；无音轨、ASR 无文本且 vision 无结果时以 `MEDIA_NO_TEXT` 进入可重试失败态，不冒充仍在排队；视频同时只加工 1 个 |
|
||||
|
||||
模型能力第二片已经落最小后端边界：
|
||||
|
||||
@@ -237,7 +237,7 @@ export function uploadKnowledgeDocAsync(data: FormData): Promise<ApiResult<Uploa
|
||||
url: '/api/knowledge/doc/upload-async',
|
||||
method: 'post',
|
||||
data,
|
||||
timeout: 60000,
|
||||
timeout: 600000,
|
||||
headers: { repeatSubmit: false }
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1078,8 +1078,8 @@ import { uploadKnowledgeDocAsync, listUploadItems, retryUploadItem, type UploadI
|
||||
import { listKnowledgeSpaces, type KnowledgeSpace } from '@/api/aihr/space';
|
||||
import { listPersonalPublishRequests, reviewPersonalPublishRequest, type PersonalPublishRequestRow } from '@/api/aihr/personal';
|
||||
|
||||
const acceptTypes = '.txt,.md,.markdown,.pdf,.doc,.docx,.xls,.xlsx,.ppt,.pptx,.jpg,.jpeg,.png,.gif,.webp,.bmp,.mp4,.mov,.avi,.mkv,.webm,.m4v,.zip';
|
||||
const supportedPattern = /\.(txt|md|markdown|pdf|doc|docx|xls|xlsx|ppt|pptx|jpg|jpeg|png|gif|webp|bmp|mp4|mov|avi|mkv|webm|m4v|zip)$/i;
|
||||
const acceptTypes = '.txt,.md,.markdown,.rtf,.pdf,.doc,.docx,.xls,.xlsx,.ppt,.pptx,.jpg,.jpeg,.png,.gif,.webp,.bmp,.mp4,.mov,.avi,.mkv,.webm,.m4v,.zip';
|
||||
const supportedPattern = /\.(txt|md|markdown|rtf|pdf|doc|docx|xls|xlsx|ppt|pptx|jpg|jpeg|png|gif|webp|bmp|mp4|mov|avi|mkv|webm|m4v|zip)$/i;
|
||||
const maxFileSize = 500 * 1024 * 1024;
|
||||
const browserBatchWarnCount = 20;
|
||||
const browserBatchWarnSize = 200 * 1024 * 1024;
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
<el-option v-for="space in manageableSpaces" :key="space.code" :label="space.name" :value="space.code" />
|
||||
</el-select>
|
||||
<el-upload
|
||||
accept=".txt,.md,.markdown,.pdf,.doc,.docx,.xls,.xlsx,.ppt,.pptx"
|
||||
accept=".txt,.md,.markdown,.rtf,.pdf,.doc,.docx,.xls,.xlsx,.ppt,.pptx"
|
||||
:auto-upload="false"
|
||||
:show-file-list="false"
|
||||
:on-change="handleDocSelected"
|
||||
|
||||
Reference in New Issue
Block a user