feat(aihr): decouple batch upload from processing via async queue
批量上传治本:POST /doc/upload-async 只暂存+入队即秒回,后台 worker
(并发2)复用 saveDocument 链路逐条解析/归类/向量化;队列表
aihr_knowledge_upload_item 状态机 0/1/2/3,完成/失败 CAS 写入,同名
文件串行防撞 attach 唯一键,卡住30分钟清扫重置,失败暂存保留72h。
新增 GET /doc/upload-items 批次查询与 POST /doc/upload-items/{id}/retry
单文件重试。资料处理页批量导入改走异步:提交即返回,新增本批次
进度面板(排队/加工中/完成/失败+进度条+单条重试),轮询防过期回写。
同步接口保留给 SOP 页单文件即时预览。
This commit is contained in:
@@ -67,3 +67,51 @@ export function uploadKnowledgeDoc(data: FormData): Promise<ApiResult<SopUploadR
|
||||
headers: { repeatSubmit: false }
|
||||
});
|
||||
}
|
||||
|
||||
export type UploadEnqueueResponse = {
|
||||
itemId: number;
|
||||
batchId: string;
|
||||
fileName: string;
|
||||
status: string;
|
||||
};
|
||||
|
||||
export type UploadItem = {
|
||||
id: number;
|
||||
batchId: string;
|
||||
fileName: string;
|
||||
category: string;
|
||||
status: number;
|
||||
statusLabel: string;
|
||||
error: string | null;
|
||||
docId: string | null;
|
||||
fragmentCount: number | null;
|
||||
updatedAt: string;
|
||||
};
|
||||
|
||||
/** 批量上传走异步队列:接口只暂存+入队即返回,解析/归类/向量化由后台 worker 处理。 */
|
||||
export function uploadKnowledgeDocAsync(data: FormData): Promise<ApiResult<UploadEnqueueResponse>> {
|
||||
return request({
|
||||
url: '/api/knowledge/doc/upload-async',
|
||||
method: 'post',
|
||||
data,
|
||||
timeout: 60000,
|
||||
headers: { repeatSubmit: false }
|
||||
});
|
||||
}
|
||||
|
||||
export function listUploadItems(batchId: string): Promise<ApiResult<UploadItem[]>> {
|
||||
return request({
|
||||
url: '/api/knowledge/doc/upload-items',
|
||||
method: 'get',
|
||||
params: { batchId },
|
||||
headers: { repeatSubmit: false }
|
||||
});
|
||||
}
|
||||
|
||||
export function retryUploadItem(id: number): Promise<ApiResult<UploadItem>> {
|
||||
return request({
|
||||
url: `/api/knowledge/doc/upload-items/${id}/retry`,
|
||||
method: 'post',
|
||||
headers: { repeatSubmit: false }
|
||||
});
|
||||
}
|
||||
|
||||
@@ -20,6 +20,45 @@
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section v-if="uploadBatch.length || enqueueFailures.length" class="panel upload-batch-panel">
|
||||
<div class="panel-title batch-title">
|
||||
<h2>本次批量上传</h2>
|
||||
<div class="batch-summary">
|
||||
<el-tag v-if="batchCounts.queued" size="small" type="info">排队 {{ batchCounts.queued }}</el-tag>
|
||||
<el-tag v-if="batchCounts.processing" size="small">加工中 {{ batchCounts.processing }}</el-tag>
|
||||
<el-tag v-if="batchCounts.done" size="small" type="success">完成 {{ batchCounts.done }}</el-tag>
|
||||
<el-tag v-if="batchCounts.failed" size="small" type="danger">失败 {{ batchCounts.failed }}</el-tag>
|
||||
<el-button v-if="!batchActive" link size="small" @click="dismissBatch">收起</el-button>
|
||||
</div>
|
||||
</div>
|
||||
<el-progress
|
||||
:percentage="batchPercent"
|
||||
:status="batchActive ? undefined : batchCounts.failed > 0 ? 'exception' : 'success'"
|
||||
:stroke-width="10"
|
||||
/>
|
||||
<ul v-if="enqueueFailures.length" class="enqueue-failures">
|
||||
<li v-for="text in enqueueFailures" :key="text">提交失败:{{ text }}</li>
|
||||
</ul>
|
||||
<el-table v-if="uploadBatch.length" :data="uploadBatch" size="small" max-height="260" class="batch-table">
|
||||
<el-table-column prop="fileName" label="文件名" min-width="220" show-overflow-tooltip />
|
||||
<el-table-column label="状态" width="96">
|
||||
<template #default="{ row }">
|
||||
<el-tag size="small" :type="batchItemTagType(row.status)">{{ row.statusLabel }}</el-tag>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column prop="fragmentCount" label="片段" width="70" />
|
||||
<el-table-column prop="error" label="失败原因" min-width="200" show-overflow-tooltip />
|
||||
<el-table-column prop="updatedAt" label="更新时间" width="130" />
|
||||
<el-table-column label="操作" width="80">
|
||||
<template #default="{ row }">
|
||||
<el-button v-if="row.status === 3" link type="danger" size="small" :loading="retryingId === row.id" @click="retryBatchItem(row)">
|
||||
重试
|
||||
</el-button>
|
||||
</template>
|
||||
</el-table-column>
|
||||
</el-table>
|
||||
</section>
|
||||
|
||||
<section class="metric-grid">
|
||||
<article v-for="metric in displayOverview.metrics" :key="metric.key" class="metric-card" :class="`metric-${metric.type}`">
|
||||
<div class="metric-icon">
|
||||
@@ -148,7 +187,11 @@
|
||||
图片 OCR
|
||||
<el-tag v-if="!displayOverview.rules.imageOcrEnabled" size="small" type="info" class="coming-soon-tag">即将支持</el-tag>
|
||||
</span>
|
||||
<el-switch :model-value="displayOverview.rules.imageOcrEnabled" disabled :active-color="displayOverview.rules.imageOcrEnabled ? '#e60012' : undefined" />
|
||||
<el-switch
|
||||
:model-value="displayOverview.rules.imageOcrEnabled"
|
||||
disabled
|
||||
:active-color="displayOverview.rules.imageOcrEnabled ? '#e60012' : undefined"
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
<span>低置信回答不编造</span>
|
||||
@@ -245,12 +288,14 @@ import {
|
||||
type LocalImportTask,
|
||||
type ProcessingOverview
|
||||
} from '@/api/aihr/processing';
|
||||
import { uploadKnowledgeDoc } from '@/api/aihr/sop';
|
||||
import { uploadKnowledgeDocAsync, listUploadItems, retryUploadItem, type UploadItem } from '@/api/aihr/sop';
|
||||
|
||||
const acceptTypes = '.txt,.md,.markdown,.pdf,.doc,.docx,.xls,.xlsx,.ppt,.pptx';
|
||||
const acceptTypes = '.txt,.md,.markdown,.pdf,.doc,.docx,.xls,.xlsx,.ppt,.pptx,.jpg,.jpeg,.png,.gif,.webp,.bmp';
|
||||
const autoCategory = '__auto__';
|
||||
const supportedPattern = /\.(txt|md|markdown|pdf|doc|docx|xls|xlsx|ppt|pptx)$/i;
|
||||
const supportedPattern = /\.(txt|md|markdown|pdf|doc|docx|xls|xlsx|ppt|pptx|jpg|jpeg|png|gif|webp|bmp)$/i;
|
||||
const maxFileSize = 100 * 1024 * 1024;
|
||||
const browserBatchWarnCount = 20;
|
||||
const browserBatchWarnSize = 200 * 1024 * 1024;
|
||||
|
||||
const fileInput = ref<HTMLInputElement>();
|
||||
const directoryInput = ref<HTMLInputElement>();
|
||||
@@ -259,13 +304,19 @@ const uploading = ref(false);
|
||||
const serverImporting = ref(false);
|
||||
const importTaskLoading = ref(false);
|
||||
const overview = ref<ProcessingOverview | null>(null);
|
||||
const overviewFailed = ref(false);
|
||||
const importTasks = ref<LocalImportTask[]>([]);
|
||||
const categoryKeyword = ref('');
|
||||
const selectedCategory = ref('住宅 SOP');
|
||||
const autoClassify = ref(true);
|
||||
const activeStatus = ref('全部');
|
||||
const fileType = ref('全部');
|
||||
const uploadBatch = ref<UploadItem[]>([]);
|
||||
const uploadBatchId = ref('');
|
||||
const enqueueFailures = ref<string[]>([]);
|
||||
const retryingId = ref<number | null>(null);
|
||||
let importPollTimer: ReturnType<typeof window.setTimeout> | undefined;
|
||||
let batchPollTimer: ReturnType<typeof window.setTimeout> | undefined;
|
||||
|
||||
const statusTabs = ['全部', '等待解析', '解析中', '已完成', '失败'];
|
||||
const fileTypes = ['全部', 'Word', 'PDF', 'PPT', 'Excel', '图片'];
|
||||
@@ -367,7 +418,34 @@ const seedOverview: ProcessingOverview = {
|
||||
}
|
||||
};
|
||||
|
||||
const displayOverview = computed(() => (overview.value && overview.value.tasks.length > 0 ? overview.value : seedOverview));
|
||||
const emptyOverview: ProcessingOverview = {
|
||||
metrics: [
|
||||
{ key: 'total', label: '资料总量', value: '0', note: '暂无导入文件', type: 'danger' },
|
||||
{ key: 'completed', label: '已完成', value: '0', note: '等待真实上传', type: 'success' },
|
||||
{ key: 'processing', label: '处理中', value: '0', note: '暂无任务', type: 'primary' },
|
||||
{ key: 'failed', label: '失败', value: '0', note: '暂无失败', type: 'warning' }
|
||||
],
|
||||
categories: [],
|
||||
tasks: [],
|
||||
pipeline: [
|
||||
{ name: '上传到 MinIO', value: '0 / 0', status: 'pending' },
|
||||
{ name: 'Tika 解析', value: '0 / 0', status: 'pending' },
|
||||
{ name: '文本切片', value: '0 片段', status: 'pending' },
|
||||
{ name: 'Embedding', value: '0 向量', status: 'pending' },
|
||||
{ name: 'Qdrant 入库', value: '0 向量', status: 'pending' },
|
||||
{ name: '可引用问答', value: '-', status: 'pending' }
|
||||
],
|
||||
events: [],
|
||||
rules: {
|
||||
parserEnabled: true,
|
||||
imageOcrEnabled: false,
|
||||
lowConfidenceRefusal: true,
|
||||
citationRequired: true,
|
||||
failedTasks: 0
|
||||
}
|
||||
};
|
||||
|
||||
const displayOverview = computed(() => overview.value ?? (overviewFailed.value ? seedOverview : emptyOverview));
|
||||
const filteredCategories = computed(() => {
|
||||
const keyword = categoryKeyword.value.trim().toLowerCase();
|
||||
return displayOverview.value.categories.filter((item) => !keyword || item.name.toLowerCase().includes(keyword));
|
||||
@@ -380,16 +458,23 @@ onMounted(async () => {
|
||||
scheduleImportPoll();
|
||||
});
|
||||
|
||||
onUnmounted(clearImportPoll);
|
||||
onUnmounted(() => {
|
||||
clearImportPoll();
|
||||
clearBatchPoll();
|
||||
// 置空批次 ID,让在途的轮询请求返回后直接丢弃,不再重新挂起定时器
|
||||
uploadBatchId.value = '';
|
||||
});
|
||||
|
||||
async function loadOverview() {
|
||||
loading.value = true;
|
||||
try {
|
||||
const { data } = await getProcessingOverview();
|
||||
overview.value = data;
|
||||
overviewFailed.value = false;
|
||||
selectedCategory.value = data.categories[0]?.name || selectedCategory.value;
|
||||
} catch {
|
||||
overview.value = null;
|
||||
overviewFailed.value = true;
|
||||
} finally {
|
||||
loading.value = false;
|
||||
}
|
||||
@@ -403,30 +488,130 @@ async function handlePickedFiles(event: Event) {
|
||||
}
|
||||
|
||||
async function uploadFiles(files: File[]) {
|
||||
const readyFiles = files.filter((file) => supportedPattern.test(file.name) && file.size <= maxFileSize);
|
||||
const readyFiles = files.filter((file) => supportedPattern.test(file.name) && file.size > 0 && file.size <= maxFileSize);
|
||||
if (readyFiles.length === 0) {
|
||||
ElMessage.warning('仅支持 100MB 内的 txt/md/pdf/Word/Excel/PPT 文件');
|
||||
ElMessage.warning('仅支持非空且 100MB 内的 txt/md/pdf/Word/Excel/PPT/图片文件');
|
||||
return;
|
||||
}
|
||||
if (readyFiles.length < files.length) {
|
||||
ElMessage.warning(`已跳过 ${files.length - readyFiles.length} 个不支持或超过 100MB 的文件`);
|
||||
ElMessage.warning(`已跳过 ${files.length - readyFiles.length} 个不支持、空文件或超过 100MB 的文件`);
|
||||
}
|
||||
|
||||
const totalSize = readyFiles.reduce((sum, file) => sum + file.size, 0);
|
||||
if (readyFiles.length > browserBatchWarnCount || totalSize > browserBatchWarnSize) {
|
||||
ElMessage.warning('浏览器批量适合少量文件;大批量资料请放到导入根目录后用服务端导入任务');
|
||||
}
|
||||
|
||||
uploading.value = true;
|
||||
clearBatchPoll();
|
||||
const batchId = `b-${window.crypto?.randomUUID?.() ?? Date.now()}`;
|
||||
uploadBatchId.value = batchId;
|
||||
uploadBatch.value = [];
|
||||
enqueueFailures.value = [];
|
||||
let queued = 0;
|
||||
try {
|
||||
for (const file of readyFiles) {
|
||||
const form = new FormData();
|
||||
form.append('file', file);
|
||||
form.append('category', categoryPayload());
|
||||
await uploadKnowledgeDoc(form);
|
||||
form.append('batchId', batchId);
|
||||
try {
|
||||
await uploadKnowledgeDocAsync(form);
|
||||
queued++;
|
||||
} catch (error) {
|
||||
enqueueFailures.value.push(`${file.name}: ${uploadFailureMessage(error)}`);
|
||||
}
|
||||
}
|
||||
if (queued > 0) {
|
||||
ElMessage.success(`已提交 ${queued} 个文件,后台加工中`);
|
||||
await pollUploadBatch();
|
||||
} else if (enqueueFailures.value.length > 0) {
|
||||
ElMessage.error(`提交失败:${enqueueFailures.value[0]}`);
|
||||
}
|
||||
ElMessage.success(`已导入 ${readyFiles.length} 个文件`);
|
||||
await loadOverview();
|
||||
} finally {
|
||||
uploading.value = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function pollUploadBatch() {
|
||||
const polledBatch = uploadBatchId.value;
|
||||
if (!polledBatch) return;
|
||||
let items: UploadItem[] | null = null;
|
||||
try {
|
||||
const { data } = await listUploadItems(polledBatch);
|
||||
items = data || [];
|
||||
} catch {
|
||||
// 单次轮询失败不清空面板,等下一轮
|
||||
}
|
||||
// 请求在途期间批次可能已被收起或替换,丢弃过期结果,避免重新挂起定时器
|
||||
if (uploadBatchId.value !== polledBatch) return;
|
||||
if (items) {
|
||||
uploadBatch.value = items;
|
||||
}
|
||||
const active = uploadBatch.value.some((item) => item.status === 0 || item.status === 1);
|
||||
clearBatchPoll();
|
||||
if (active || uploadBatch.value.length === 0) {
|
||||
batchPollTimer = window.setTimeout(() => void pollUploadBatch(), 2000);
|
||||
} else {
|
||||
await loadOverview();
|
||||
}
|
||||
}
|
||||
|
||||
function clearBatchPoll() {
|
||||
if (batchPollTimer) {
|
||||
window.clearTimeout(batchPollTimer);
|
||||
batchPollTimer = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function dismissBatch() {
|
||||
clearBatchPoll();
|
||||
uploadBatch.value = [];
|
||||
uploadBatchId.value = '';
|
||||
enqueueFailures.value = [];
|
||||
}
|
||||
|
||||
async function retryBatchItem(item: UploadItem) {
|
||||
retryingId.value = item.id;
|
||||
try {
|
||||
await retryUploadItem(item.id);
|
||||
ElMessage.success(`已重新排队:${item.fileName}`);
|
||||
await pollUploadBatch();
|
||||
} catch (error) {
|
||||
ElMessage.error(uploadFailureMessage(error));
|
||||
} finally {
|
||||
retryingId.value = null;
|
||||
}
|
||||
}
|
||||
|
||||
const batchCounts = computed(() => ({
|
||||
queued: uploadBatch.value.filter((item) => item.status === 0).length,
|
||||
processing: uploadBatch.value.filter((item) => item.status === 1).length,
|
||||
done: uploadBatch.value.filter((item) => item.status === 2).length,
|
||||
failed: uploadBatch.value.filter((item) => item.status === 3).length
|
||||
}));
|
||||
|
||||
const batchActive = computed(() => batchCounts.value.queued > 0 || batchCounts.value.processing > 0);
|
||||
|
||||
const batchPercent = computed(() => {
|
||||
const total = uploadBatch.value.length;
|
||||
if (total === 0) return 0;
|
||||
return Math.round(((batchCounts.value.done + batchCounts.value.failed) / total) * 100);
|
||||
});
|
||||
|
||||
function batchItemTagType(status: number) {
|
||||
if (status === 2) return 'success';
|
||||
if (status === 3) return 'danger';
|
||||
if (status === 1) return 'primary';
|
||||
return 'info';
|
||||
}
|
||||
|
||||
function uploadFailureMessage(error: unknown) {
|
||||
if (error instanceof Error && error.message) return error.message;
|
||||
if (typeof error === 'string' && error) return error;
|
||||
return '上传失败';
|
||||
}
|
||||
|
||||
async function importServerDirectory() {
|
||||
let directory = '.';
|
||||
try {
|
||||
@@ -741,6 +926,33 @@ function fileBadgeClass(type: string) {
|
||||
padding: 16px;
|
||||
}
|
||||
|
||||
.upload-batch-panel {
|
||||
margin-bottom: 14px;
|
||||
|
||||
.batch-title {
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
.batch-summary {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.batch-table {
|
||||
margin-top: 12px;
|
||||
}
|
||||
|
||||
.enqueue-failures {
|
||||
margin: 10px 0 0;
|
||||
padding-left: 18px;
|
||||
color: #d4380d;
|
||||
font-size: 12px;
|
||||
line-height: 1.8;
|
||||
word-break: break-all;
|
||||
}
|
||||
}
|
||||
|
||||
.panel-title {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
|
||||
Reference in New Issue
Block a user