fix(personal): complete scanned PDF OCR gate
This commit is contained in:
+12
-1
@@ -185,7 +185,18 @@ public class PersonalIngestionWorker {
|
||||
try {
|
||||
StoredObject stored = tenantRunner.execute(item.tenantId(), () -> objectReader.read(
|
||||
item.ossId(), ownerObjectPrefix(item), item.ownerUserId(), maxInputBytes));
|
||||
ParsedDocument document = parser.parse(stored.fileName(), item.mimeType(), stored.bytes());
|
||||
ParsedDocument document;
|
||||
try {
|
||||
document = parser.parse(stored.fileName(), item.mimeType(), stored.bytes());
|
||||
} catch (KnowledgeDocumentParser.ParseException exception) {
|
||||
if (exception.failure() == KnowledgeDocumentParser.Failure.EMPTY && isPdf(item, stored)) {
|
||||
pdfOcrHandler.enqueue(new PersonalPdfOcrService.OcrItem(item.id(), item.tenantId(),
|
||||
item.spaceId(), item.ownerUserId(), item.ossId(), item.tagsJson(), attemptVersion),
|
||||
stored.bytes());
|
||||
return true;
|
||||
}
|
||||
throw exception;
|
||||
}
|
||||
List<String> chunks = document.chunks(chunkSize, chunkOverlap);
|
||||
if (chunks.isEmpty()) {
|
||||
if (isPdf(item, stored)) {
|
||||
|
||||
+30
-3
@@ -352,14 +352,21 @@ public class PersonalPdfOcrService {
|
||||
where tenant_id = ? and owner_user_id = ? and item_id = ? and status = 'FAILED'
|
||||
order by page_number
|
||||
""", Integer.class, job.tenantId(), job.ownerUserId(), job.itemId());
|
||||
List<String> failureCodes = pageTexts.isEmpty() ? jdbcTemplate.queryForList("""
|
||||
select distinct last_error_code from aihr_personal_ocr_page
|
||||
where tenant_id = ? and owner_user_id = ? and item_id = ?
|
||||
and status = 'FAILED' and last_error_code is not null
|
||||
order by last_error_code
|
||||
""", String.class, job.tenantId(), job.ownerUserId(), job.itemId()) : List.of();
|
||||
transactions.execute(status -> {
|
||||
if (pageTexts.isEmpty()) {
|
||||
String failureCode = terminalFailureCode(failureCodes);
|
||||
jdbcTemplate.update("""
|
||||
update aihr_personal_item
|
||||
set status = 'FAILED', error_code = 'PERSONAL_OCR_EMPTY',
|
||||
error_message = '扫描PDF未识别到可用文字', update_time = now()
|
||||
set status = 'FAILED', error_code = ?, error_message = ?, update_time = now()
|
||||
where tenant_id = ? and owner_user_id = ? and id = ? and status = 'PARSING'
|
||||
""", job.tenantId(), job.ownerUserId(), job.itemId());
|
||||
""", failureCode, terminalFailureMessage(failureCode),
|
||||
job.tenantId(), job.ownerUserId(), job.itemId());
|
||||
} else {
|
||||
publishFragments(job, String.join("\n\n", pageTexts), failedPages);
|
||||
}
|
||||
@@ -374,6 +381,26 @@ public class PersonalPdfOcrService {
|
||||
});
|
||||
}
|
||||
|
||||
private static String terminalFailureCode(List<String> failureCodes) {
|
||||
if (failureCodes.size() == 1 && !"PERSONAL_OCR_PAGE_EMPTY".equals(failureCodes.get(0))) {
|
||||
return failureCodes.get(0);
|
||||
}
|
||||
if (failureCodes.stream().anyMatch(code -> code != null && !"PERSONAL_OCR_PAGE_EMPTY".equals(code))) {
|
||||
return "PERSONAL_OCR_UNAVAILABLE";
|
||||
}
|
||||
return "PERSONAL_OCR_EMPTY";
|
||||
}
|
||||
|
||||
private static String terminalFailureMessage(String failureCode) {
|
||||
return switch (failureCode) {
|
||||
case "PERSONAL_OCR_MODEL_UNAVAILABLE" -> "未配置支持图片的视觉模型，请配置后重试失败页";
|
||||
case "PERSONAL_OCR_RUNTIME_DISABLED" -> "扫描PDF识别服务已关闭，请启用后重试失败页";
|
||||
case "PERSONAL_OCR_PROVIDER_FAILED", "PERSONAL_OCR_RESPONSE_INVALID", "PERSONAL_OCR_UNAVAILABLE" ->
|
||||
"扫描PDF识别服务暂不可用，请稍后重试失败页";
|
||||
default -> "扫描PDF未识别到可用文字";
|
||||
};
|
||||
}
|
||||
|
||||
private void publishFragments(OcrJob job, String text, List<Integer> failedPages) {
|
||||
List<String> chunks = new ParsedDocument(text, "application/pdf", Map.of()).chunks(800, 120);
|
||||
jdbcTemplate.update("""
|
||||
|
||||
+2
-1
@@ -44,7 +44,8 @@ class PersonalIngestionWorkerTest {
|
||||
when(jdbc.queryForList(contains("status = 'QUEUED'"))).thenReturn(List.of(item()));
|
||||
when(jdbc.update(contains("status = 'PARSING'"), eq("000000"), eq(101L), eq(9L), eq(0))).thenReturn(1);
|
||||
when(parser.parse(eq("sample.pdf"), eq("text/plain"), any(byte[].class)))
|
||||
.thenReturn(new ParsedDocument("", "application/pdf", Map.of()));
|
||||
.thenThrow(new KnowledgeDocumentParser.ParseException(
|
||||
KnowledgeDocumentParser.Failure.EMPTY, "document contains no text"));
|
||||
AtomicReference<org.dromara.aihr.personal.service.PersonalPdfOcrService.OcrItem> handedOff =
|
||||
new AtomicReference<>();
|
||||
PersonalIngestionWorker worker = PersonalIngestionWorker.forTestWithPdfOcr(
|
||||
|
||||
+28
@@ -165,6 +165,34 @@ class PersonalPdfOcrServiceTest {
|
||||
eq("000000"), eq(101L), eq(9L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void allPagesRejectedByProviderExposeModelFailureInsteadOfEmptyDocument() {
|
||||
JdbcTemplate jdbc = mock(JdbcTemplate.class);
|
||||
Map<String, Object> job = Map.ofEntries(
|
||||
Map.entry("id", 5L), Map.entry("tenant_id", "000000"), Map.entry("owner_user_id", 101L),
|
||||
Map.entry("item_id", 9L), Map.entry("space_id", 7L), Map.entry("oss_id", 88L),
|
||||
Map.entry("tags_json", "[]"), Map.entry("total_pages", 8), Map.entry("next_page", 9),
|
||||
Map.entry("status", "RETRY"));
|
||||
when(jdbc.queryForList(contains("from aihr_personal_ocr_job"))).thenReturn(List.of(job));
|
||||
when(jdbc.update(contains("set status = 'RUNNING'"), eq(5L))).thenReturn(1);
|
||||
when(jdbc.queryForList(contains("attempt_count < 3"), eq(Integer.class),
|
||||
eq("000000"), eq(101L), eq(9L), eq(20))).thenReturn(List.of());
|
||||
when(jdbc.queryForList(contains("select ocr_text"), eq(String.class),
|
||||
eq("000000"), eq(101L), eq(9L))).thenReturn(List.of());
|
||||
when(jdbc.queryForList(contains("select page_number"), eq(Integer.class),
|
||||
eq("000000"), eq(101L), eq(9L))).thenReturn(List.of(1, 2, 3, 4, 5, 6, 7, 8));
|
||||
when(jdbc.queryForList(contains("select distinct last_error_code"), eq(String.class),
|
||||
eq("000000"), eq(101L), eq(9L))).thenReturn(List.of("PERSONAL_OCR_PROVIDER_FAILED"));
|
||||
PersonalPdfOcrService service = PersonalPdfOcrService.forTest(
|
||||
jdbc, mock(PersonalPdfPageRenderer.class), mock(PersonalVisionOcrService.class),
|
||||
immediateTransactions(), ignored -> new byte[] {1});
|
||||
|
||||
assertTrue(service.processNextBatch());
|
||||
|
||||
verify(jdbc).update(contains("set status = 'FAILED', error_code = ?"),
|
||||
eq("PERSONAL_OCR_PROVIDER_FAILED"), any(String.class), eq("000000"), eq(101L), eq(9L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void recoversExpiredRunningLease() {
|
||||
JdbcTemplate jdbc = mock(JdbcTemplate.class);
|
||||
|
||||
@@ -23,11 +23,15 @@ PUBLIC_URL="${AIHR_PERSONAL_SMOKE_PUBLIC_URL:-https://example.com/}"
|
||||
PUBLIC_EXPECTED_URL="${AIHR_PERSONAL_SMOKE_EXPECTED_PUBLIC_URL:-}"
|
||||
[[ -n "${AIHR_PERSONAL_SMOKE_PUBLIC_URL:-}" ]] || PUBLIC_EXPECTED_URL="https://example.com/"
|
||||
PUBLIC_QUERY="${AIHR_PERSONAL_SMOKE_PUBLIC_QUERY:-Example Domain}"
|
||||
SCANNED_PDF="${AIHR_PERSONAL_SCANNED_PDF:-}"
|
||||
SCANNED_QUERY="${AIHR_PERSONAL_SCANNED_QUERY:-证书管理办法}"
|
||||
SCANNED_TITLE="$TITLE-scanned-pdf"
|
||||
TMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/wygj-personal-smoke.XXXXXX")"
|
||||
chmod 700 "$TMP_ROOT"
|
||||
|
||||
TOKEN_A="" CLIENT_A="" TOKEN_B="" CLIENT_B=""
|
||||
USER_A="" USER_B="" SESSION_ID=""
|
||||
SCANNED_ITEM_ID=""
|
||||
OWN_PHONE_A=0 OWN_PHONE_B=0 CLEANED=0
|
||||
HTTP_STATUS="" HTTP_BODY="" LOGIN_TOKEN="" LOGIN_CLIENT=""
|
||||
TEMP_FILES=()
|
||||
@@ -115,6 +119,8 @@ cleanup_once() {
|
||||
mysql "delete from sys_oss where oss_id=$oss_id and service='personal-minio' and file_name='$object_key'" >/dev/null 2>&1 || true
|
||||
fi
|
||||
mysql "delete from aihr_personal_fragment where tenant_id='000000' and owner_user_id=$USER_A and item_id=$item_id;
|
||||
delete from aihr_personal_ocr_page where tenant_id='000000' and owner_user_id=$USER_A and item_id=$item_id;
|
||||
delete from aihr_personal_ocr_job where tenant_id='000000' and owner_user_id=$USER_A and item_id=$item_id;
|
||||
delete from aihr_personal_cleanup_job where tenant_id='000000' and owner_user_id=$USER_A and item_id=$item_id;
|
||||
delete from aihr_personal_item where tenant_id='000000' and owner_user_id=$USER_A and id=$item_id and title like '$TITLE-%';" >/dev/null 2>&1 || true
|
||||
done
|
||||
@@ -254,6 +260,21 @@ wait_ready() {
|
||||
fail "$label did not become READY lastStatus=$status"
|
||||
}
|
||||
|
||||
wait_scanned_ready() {
|
||||
local item_id="$1" status="" ocr_status=""
|
||||
for _ in {1..180}; do
|
||||
request GET "/api/aihr/personal-assistant/items/$item_id" "$TOKEN_A" "$CLIENT_A"
|
||||
expect_success "scanned PDF detail poll"
|
||||
status="$(jq -r '.data.status // empty' <<<"$HTTP_BODY")"
|
||||
ocr_status="$(jq -r '.data.ocr.status // empty' <<<"$HTTP_BODY")"
|
||||
[[ "$status" == READY ]] && return 0
|
||||
[[ "$status" != FAILED ]] \
|
||||
|| fail "scanned PDF OCR failed code=$(jq -r '.data.errorCode // empty' <<<"$HTTP_BODY") ocrStatus=$ocr_status failedPages=$(jq -c '.data.ocr.failedPageNumbers // []' <<<"$HTTP_BODY")"
|
||||
sleep 1
|
||||
done
|
||||
fail "scanned PDF did not become READY lastStatus=$status ocrStatus=$ocr_status"
|
||||
}
|
||||
|
||||
assert_anonymous_private() {
|
||||
local url="$1" label="$2" tmp status
|
||||
tmp="$(mktemp "$TMP_ROOT/anonymous.XXXXXX")"
|
||||
@@ -368,6 +389,8 @@ request GET /auth/tenant/list
|
||||
expect_success "backend health"
|
||||
docker ps --format '{{.Names}}' | grep -qx "$DB_CONTAINER" || fail "database container not running: $DB_CONTAINER"
|
||||
mysql "select 1 from aihr_personal_item limit 1" >/dev/null || fail "personal schema missing; run COMPOSE_PROJECT_NAME=wygj ./scripts/reset-dev-db.sh"
|
||||
mysql "select 1 from aihr_personal_ocr_job limit 1" >/dev/null || fail "personal OCR job schema missing; import aihr_personal_knowledge_mysql8.sql"
|
||||
mysql "select 1 from aihr_personal_ocr_page limit 1" >/dev/null || fail "personal OCR page schema missing; import aihr_personal_knowledge_mysql8.sql"
|
||||
expect_code "$(mysql "select access_policy from sys_oss_config where tenant_id='000000' and config_key='minio'")" 1 "shared minio policy unchanged"
|
||||
expect_code "$(mysql "select concat(bucket_name,':',access_policy) from sys_oss_config where tenant_id='000000' and config_key='personal-minio'")" "ruoyi-personal:0" "personal private storage config"
|
||||
expect_code "$(mysql "select count(*) from sys_user where phonenumber in ('$PHONE_A','$PHONE_B')")" 0 "unique smoke users must not preexist"
|
||||
@@ -430,6 +453,31 @@ assert_search_hit "$TEXT_ITEM_ID" "$TEXT_QUERY" TEXT text
|
||||
assert_search_hit "$PDF_ITEM_ID" "$PDF_QUERY" FILE PDF
|
||||
assert_search_hit "$URL_ITEM_ID" "$PUBLIC_QUERY" URL URL
|
||||
|
||||
if [[ -n "$SCANNED_PDF" ]]; then
|
||||
[[ -f "$SCANNED_PDF" && -s "$SCANNED_PDF" ]] || fail "AIHR_PERSONAL_SCANNED_PDF is not a readable file: $SCANNED_PDF"
|
||||
request_file /api/aihr/personal-assistant/items/file "$TOKEN_A" "$CLIENT_A" "$SCANNED_PDF" "$SCANNED_TITLE"
|
||||
expect_success "A create scanned PDF item"
|
||||
SCANNED_ITEM_ID="$(jq -er '.data.itemId | tostring' <<<"$HTTP_BODY")"
|
||||
append_item_metadata "$SCANNED_ITEM_ID" "$SCANNED_TITLE"
|
||||
wait_scanned_ready "$SCANNED_ITEM_ID"
|
||||
[[ "$(jq -r '.data.ocr.required // false' <<<"$HTTP_BODY")" == true ]] \
|
||||
|| fail "scanned PDF did not enter OCR pipeline"
|
||||
expect_code "$(jq -r '.data.ocr.processedPages // 0' <<<"$HTTP_BODY")" \
|
||||
"$(jq -r '.data.ocr.totalPages // -1' <<<"$HTTP_BODY")" "scanned PDF processed page count"
|
||||
assert_search_hit "$SCANNED_ITEM_ID" "$SCANNED_QUERY" FILE "scanned PDF"
|
||||
request DELETE "/api/aihr/personal-assistant/items/$SCANNED_ITEM_ID" "$TOKEN_A" "$CLIENT_A"
|
||||
expect_success "A delete scanned PDF item"
|
||||
for _ in {1..75}; do
|
||||
scanned_cleanup_done="$(mysql "select count(*) from aihr_personal_cleanup_job
|
||||
where tenant_id='000000' and owner_user_id=$USER_A and item_id=$SCANNED_ITEM_ID and status='DONE'")"
|
||||
[[ "$scanned_cleanup_done" == 1 ]] && break
|
||||
sleep 1
|
||||
done
|
||||
expect_code "$scanned_cleanup_done" 1 "scanned PDF cleanup job completed"
|
||||
expect_code "$(mysql "select count(*) from aihr_personal_ocr_page where tenant_id='000000' and owner_user_id=$USER_A and item_id=$SCANNED_ITEM_ID")" 0 "scanned PDF OCR page residual"
|
||||
expect_code "$(mysql "select count(*) from aihr_personal_ocr_job where tenant_id='000000' and owner_user_id=$USER_A and item_id=$SCANNED_ITEM_ID")" 0 "scanned PDF OCR job residual"
|
||||
fi
|
||||
|
||||
assert_single_item_ask "$TEXT_ITEM_ID" "$TEXT_QUERY" TEXT text
|
||||
assert_single_item_ask "$PDF_ITEM_ID" "$PDF_QUERY" FILE PDF
|
||||
assert_single_item_ask "$URL_ITEM_ID" "$PUBLIC_QUERY" URL URL
|
||||
|
||||
Reference in New Issue
Block a user