diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/personal/service/PersonalIngestionWorker.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/personal/service/PersonalIngestionWorker.java index 67106064..d310e5e6 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/personal/service/PersonalIngestionWorker.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/personal/service/PersonalIngestionWorker.java @@ -185,7 +185,18 @@ public class PersonalIngestionWorker { try { StoredObject stored = tenantRunner.execute(item.tenantId(), () -> objectReader.read( item.ossId(), ownerObjectPrefix(item), item.ownerUserId(), maxInputBytes)); - ParsedDocument document = parser.parse(stored.fileName(), item.mimeType(), stored.bytes()); + ParsedDocument document; + try { + document = parser.parse(stored.fileName(), item.mimeType(), stored.bytes()); + } catch (KnowledgeDocumentParser.ParseException exception) { + if (exception.failure() == KnowledgeDocumentParser.Failure.EMPTY && isPdf(item, stored)) { + pdfOcrHandler.enqueue(new PersonalPdfOcrService.OcrItem(item.id(), item.tenantId(), + item.spaceId(), item.ownerUserId(), item.ossId(), item.tagsJson(), attemptVersion), + stored.bytes()); + return true; + } + throw exception; + } List chunks = document.chunks(chunkSize, chunkOverlap); if (chunks.isEmpty()) { if (isPdf(item, stored)) { diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/personal/service/PersonalPdfOcrService.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/personal/service/PersonalPdfOcrService.java index fb7e10db..99ec20ac 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/personal/service/PersonalPdfOcrService.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/personal/service/PersonalPdfOcrService.java @@ -352,14 +352,21 @@ public class PersonalPdfOcrService { where tenant_id = ? and owner_user_id = ? and item_id = ? and status = 'FAILED' order by page_number """, Integer.class, job.tenantId(), job.ownerUserId(), job.itemId()); + List failureCodes = pageTexts.isEmpty() ? jdbcTemplate.queryForList(""" + select distinct last_error_code from aihr_personal_ocr_page + where tenant_id = ? and owner_user_id = ? and item_id = ? + and status = 'FAILED' and last_error_code is not null + order by last_error_code + """, String.class, job.tenantId(), job.ownerUserId(), job.itemId()) : List.of(); transactions.execute(status -> { if (pageTexts.isEmpty()) { + String failureCode = terminalFailureCode(failureCodes); jdbcTemplate.update(""" update aihr_personal_item - set status = 'FAILED', error_code = 'PERSONAL_OCR_EMPTY', - error_message = '扫描PDF未识别到可用文字', update_time = now() + set status = 'FAILED', error_code = ?, error_message = ?, update_time = now() where tenant_id = ? and owner_user_id = ? and id = ? and status = 'PARSING' - """, job.tenantId(), job.ownerUserId(), job.itemId()); + """, failureCode, terminalFailureMessage(failureCode), + job.tenantId(), job.ownerUserId(), job.itemId()); } else { publishFragments(job, String.join("\n\n", pageTexts), failedPages); } @@ -374,6 +381,26 @@ public class PersonalPdfOcrService { }); } + private static String terminalFailureCode(List failureCodes) { + if (failureCodes.size() == 1 && !"PERSONAL_OCR_PAGE_EMPTY".equals(failureCodes.get(0))) { + return failureCodes.get(0); + } + if (failureCodes.stream().anyMatch(code -> code != null && !"PERSONAL_OCR_PAGE_EMPTY".equals(code))) { + return "PERSONAL_OCR_UNAVAILABLE"; + } + return "PERSONAL_OCR_EMPTY"; + } + + private static String terminalFailureMessage(String failureCode) { + return switch (failureCode) { + case "PERSONAL_OCR_MODEL_UNAVAILABLE" -> "未配置支持图片的视觉模型,请配置后重试失败页"; + case "PERSONAL_OCR_RUNTIME_DISABLED" -> "扫描PDF识别服务已关闭,请启用后重试失败页"; + case "PERSONAL_OCR_PROVIDER_FAILED", "PERSONAL_OCR_RESPONSE_INVALID", "PERSONAL_OCR_UNAVAILABLE" -> + "扫描PDF识别服务暂不可用,请稍后重试失败页"; + default -> "扫描PDF未识别到可用文字"; + }; + } + private void publishFragments(OcrJob job, String text, List failedPages) { List chunks = new ParsedDocument(text, "application/pdf", Map.of()).chunks(800, 120); jdbcTemplate.update(""" diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/personal/PersonalIngestionWorkerTest.java b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/personal/PersonalIngestionWorkerTest.java index f0ad1fea..9af28536 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/personal/PersonalIngestionWorkerTest.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/personal/PersonalIngestionWorkerTest.java @@ -44,7 +44,8 @@ class PersonalIngestionWorkerTest { when(jdbc.queryForList(contains("status = 'QUEUED'"))).thenReturn(List.of(item())); when(jdbc.update(contains("status = 'PARSING'"), eq("000000"), eq(101L), eq(9L), eq(0))).thenReturn(1); when(parser.parse(eq("sample.pdf"), eq("text/plain"), any(byte[].class))) - .thenReturn(new ParsedDocument("", "application/pdf", Map.of())); + .thenThrow(new KnowledgeDocumentParser.ParseException( + KnowledgeDocumentParser.Failure.EMPTY, "document contains no text")); AtomicReference handedOff = new AtomicReference<>(); PersonalIngestionWorker worker = PersonalIngestionWorker.forTestWithPdfOcr( diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/personal/PersonalPdfOcrServiceTest.java b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/personal/PersonalPdfOcrServiceTest.java index f865cd33..1de58456 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/personal/PersonalPdfOcrServiceTest.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/personal/PersonalPdfOcrServiceTest.java @@ -165,6 +165,34 @@ class PersonalPdfOcrServiceTest { eq("000000"), eq(101L), eq(9L)); } + @Test + void allPagesRejectedByProviderExposeModelFailureInsteadOfEmptyDocument() { + JdbcTemplate jdbc = mock(JdbcTemplate.class); + Map job = Map.ofEntries( + Map.entry("id", 5L), Map.entry("tenant_id", "000000"), Map.entry("owner_user_id", 101L), + Map.entry("item_id", 9L), Map.entry("space_id", 7L), Map.entry("oss_id", 88L), + Map.entry("tags_json", "[]"), Map.entry("total_pages", 8), Map.entry("next_page", 9), + Map.entry("status", "RETRY")); + when(jdbc.queryForList(contains("from aihr_personal_ocr_job"))).thenReturn(List.of(job)); + when(jdbc.update(contains("set status = 'RUNNING'"), eq(5L))).thenReturn(1); + when(jdbc.queryForList(contains("attempt_count < 3"), eq(Integer.class), + eq("000000"), eq(101L), eq(9L), eq(20))).thenReturn(List.of()); + when(jdbc.queryForList(contains("select ocr_text"), eq(String.class), + eq("000000"), eq(101L), eq(9L))).thenReturn(List.of()); + when(jdbc.queryForList(contains("select page_number"), eq(Integer.class), + eq("000000"), eq(101L), eq(9L))).thenReturn(List.of(1, 2, 3, 4, 5, 6, 7, 8)); + when(jdbc.queryForList(contains("select distinct last_error_code"), eq(String.class), + eq("000000"), eq(101L), eq(9L))).thenReturn(List.of("PERSONAL_OCR_PROVIDER_FAILED")); + PersonalPdfOcrService service = PersonalPdfOcrService.forTest( + jdbc, mock(PersonalPdfPageRenderer.class), mock(PersonalVisionOcrService.class), + immediateTransactions(), ignored -> new byte[] {1}); + + assertTrue(service.processNextBatch()); + + verify(jdbc).update(contains("set status = 'FAILED', error_code = ?"), + eq("PERSONAL_OCR_PROVIDER_FAILED"), any(String.class), eq("000000"), eq(101L), eq(9L)); + } + @Test void recoversExpiredRunningLease() { JdbcTemplate jdbc = mock(JdbcTemplate.class); diff --git a/scripts/personal-assistant-smoke.sh b/scripts/personal-assistant-smoke.sh index 7d0e6eb1..e7a064fa 100755 --- a/scripts/personal-assistant-smoke.sh +++ b/scripts/personal-assistant-smoke.sh @@ -23,11 +23,15 @@ PUBLIC_URL="${AIHR_PERSONAL_SMOKE_PUBLIC_URL:-https://example.com/}" PUBLIC_EXPECTED_URL="${AIHR_PERSONAL_SMOKE_EXPECTED_PUBLIC_URL:-}" [[ -n "${AIHR_PERSONAL_SMOKE_PUBLIC_URL:-}" ]] || PUBLIC_EXPECTED_URL="https://example.com/" PUBLIC_QUERY="${AIHR_PERSONAL_SMOKE_PUBLIC_QUERY:-Example Domain}" +SCANNED_PDF="${AIHR_PERSONAL_SCANNED_PDF:-}" +SCANNED_QUERY="${AIHR_PERSONAL_SCANNED_QUERY:-证书管理办法}" +SCANNED_TITLE="$TITLE-scanned-pdf" TMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/wygj-personal-smoke.XXXXXX")" chmod 700 "$TMP_ROOT" TOKEN_A="" CLIENT_A="" TOKEN_B="" CLIENT_B="" USER_A="" USER_B="" SESSION_ID="" +SCANNED_ITEM_ID="" OWN_PHONE_A=0 OWN_PHONE_B=0 CLEANED=0 HTTP_STATUS="" HTTP_BODY="" LOGIN_TOKEN="" LOGIN_CLIENT="" TEMP_FILES=() @@ -115,6 +119,8 @@ cleanup_once() { mysql "delete from sys_oss where oss_id=$oss_id and service='personal-minio' and file_name='$object_key'" >/dev/null 2>&1 || true fi mysql "delete from aihr_personal_fragment where tenant_id='000000' and owner_user_id=$USER_A and item_id=$item_id; + delete from aihr_personal_ocr_page where tenant_id='000000' and owner_user_id=$USER_A and item_id=$item_id; + delete from aihr_personal_ocr_job where tenant_id='000000' and owner_user_id=$USER_A and item_id=$item_id; delete from aihr_personal_cleanup_job where tenant_id='000000' and owner_user_id=$USER_A and item_id=$item_id; delete from aihr_personal_item where tenant_id='000000' and owner_user_id=$USER_A and id=$item_id and title like '$TITLE-%';" >/dev/null 2>&1 || true done @@ -254,6 +260,21 @@ wait_ready() { fail "$label did not become READY lastStatus=$status" } +wait_scanned_ready() { + local item_id="$1" status="" ocr_status="" + for _ in {1..180}; do + request GET "/api/aihr/personal-assistant/items/$item_id" "$TOKEN_A" "$CLIENT_A" + expect_success "scanned PDF detail poll" + status="$(jq -r '.data.status // empty' <<<"$HTTP_BODY")" + ocr_status="$(jq -r '.data.ocr.status // empty' <<<"$HTTP_BODY")" + [[ "$status" == READY ]] && return 0 + [[ "$status" != FAILED ]] \ + || fail "scanned PDF OCR failed code=$(jq -r '.data.errorCode // empty' <<<"$HTTP_BODY") ocrStatus=$ocr_status failedPages=$(jq -c '.data.ocr.failedPageNumbers // []' <<<"$HTTP_BODY")" + sleep 1 + done + fail "scanned PDF did not become READY lastStatus=$status ocrStatus=$ocr_status" +} + assert_anonymous_private() { local url="$1" label="$2" tmp status tmp="$(mktemp "$TMP_ROOT/anonymous.XXXXXX")" @@ -368,6 +389,8 @@ request GET /auth/tenant/list expect_success "backend health" docker ps --format '{{.Names}}' | grep -qx "$DB_CONTAINER" || fail "database container not running: $DB_CONTAINER" mysql "select 1 from aihr_personal_item limit 1" >/dev/null || fail "personal schema missing; run COMPOSE_PROJECT_NAME=wygj ./scripts/reset-dev-db.sh" +mysql "select 1 from aihr_personal_ocr_job limit 1" >/dev/null || fail "personal OCR job schema missing; import aihr_personal_knowledge_mysql8.sql" +mysql "select 1 from aihr_personal_ocr_page limit 1" >/dev/null || fail "personal OCR page schema missing; import aihr_personal_knowledge_mysql8.sql" expect_code "$(mysql "select access_policy from sys_oss_config where tenant_id='000000' and config_key='minio'")" 1 "shared minio policy unchanged" expect_code "$(mysql "select concat(bucket_name,':',access_policy) from sys_oss_config where tenant_id='000000' and config_key='personal-minio'")" "ruoyi-personal:0" "personal private storage config" expect_code "$(mysql "select count(*) from sys_user where phonenumber in ('$PHONE_A','$PHONE_B')")" 0 "unique smoke users must not preexist" @@ -430,6 +453,31 @@ assert_search_hit "$TEXT_ITEM_ID" "$TEXT_QUERY" TEXT text assert_search_hit "$PDF_ITEM_ID" "$PDF_QUERY" FILE PDF assert_search_hit "$URL_ITEM_ID" "$PUBLIC_QUERY" URL URL +if [[ -n "$SCANNED_PDF" ]]; then + [[ -f "$SCANNED_PDF" && -s "$SCANNED_PDF" ]] || fail "AIHR_PERSONAL_SCANNED_PDF is not a readable file: $SCANNED_PDF" + request_file /api/aihr/personal-assistant/items/file "$TOKEN_A" "$CLIENT_A" "$SCANNED_PDF" "$SCANNED_TITLE" + expect_success "A create scanned PDF item" + SCANNED_ITEM_ID="$(jq -er '.data.itemId | tostring' <<<"$HTTP_BODY")" + append_item_metadata "$SCANNED_ITEM_ID" "$SCANNED_TITLE" + wait_scanned_ready "$SCANNED_ITEM_ID" + [[ "$(jq -r '.data.ocr.required // false' <<<"$HTTP_BODY")" == true ]] \ + || fail "scanned PDF did not enter OCR pipeline" + expect_code "$(jq -r '.data.ocr.processedPages // 0' <<<"$HTTP_BODY")" \ + "$(jq -r '.data.ocr.totalPages // -1' <<<"$HTTP_BODY")" "scanned PDF processed page count" + assert_search_hit "$SCANNED_ITEM_ID" "$SCANNED_QUERY" FILE "scanned PDF" + request DELETE "/api/aihr/personal-assistant/items/$SCANNED_ITEM_ID" "$TOKEN_A" "$CLIENT_A" + expect_success "A delete scanned PDF item" + for _ in {1..75}; do + scanned_cleanup_done="$(mysql "select count(*) from aihr_personal_cleanup_job + where tenant_id='000000' and owner_user_id=$USER_A and item_id=$SCANNED_ITEM_ID and status='DONE'")" + [[ "$scanned_cleanup_done" == 1 ]] && break + sleep 1 + done + expect_code "$scanned_cleanup_done" 1 "scanned PDF cleanup job completed" + expect_code "$(mysql "select count(*) from aihr_personal_ocr_page where tenant_id='000000' and owner_user_id=$USER_A and item_id=$SCANNED_ITEM_ID")" 0 "scanned PDF OCR page residual" + expect_code "$(mysql "select count(*) from aihr_personal_ocr_job where tenant_id='000000' and owner_user_id=$USER_A and item_id=$SCANNED_ITEM_ID")" 0 "scanned PDF OCR job residual" +fi + assert_single_item_ask "$TEXT_ITEM_ID" "$TEXT_QUERY" TEXT text assert_single_item_ask "$PDF_ITEM_ID" "$PDF_QUERY" FILE PDF assert_single_item_ask "$URL_ITEM_ID" "$PUBLIC_QUERY" URL URL