fix(personal): complete scanned PDF OCR gate
This commit is contained in:
+12
-1
@@ -185,7 +185,18 @@ public class PersonalIngestionWorker {
|
||||
try {
|
||||
StoredObject stored = tenantRunner.execute(item.tenantId(), () -> objectReader.read(
|
||||
item.ossId(), ownerObjectPrefix(item), item.ownerUserId(), maxInputBytes));
|
||||
ParsedDocument document = parser.parse(stored.fileName(), item.mimeType(), stored.bytes());
|
||||
ParsedDocument document;
|
||||
try {
|
||||
document = parser.parse(stored.fileName(), item.mimeType(), stored.bytes());
|
||||
} catch (KnowledgeDocumentParser.ParseException exception) {
|
||||
if (exception.failure() == KnowledgeDocumentParser.Failure.EMPTY && isPdf(item, stored)) {
|
||||
pdfOcrHandler.enqueue(new PersonalPdfOcrService.OcrItem(item.id(), item.tenantId(),
|
||||
item.spaceId(), item.ownerUserId(), item.ossId(), item.tagsJson(), attemptVersion),
|
||||
stored.bytes());
|
||||
return true;
|
||||
}
|
||||
throw exception;
|
||||
}
|
||||
List<String> chunks = document.chunks(chunkSize, chunkOverlap);
|
||||
if (chunks.isEmpty()) {
|
||||
if (isPdf(item, stored)) {
|
||||
|
||||
+30
-3
@@ -352,14 +352,21 @@ public class PersonalPdfOcrService {
|
||||
where tenant_id = ? and owner_user_id = ? and item_id = ? and status = 'FAILED'
|
||||
order by page_number
|
||||
""", Integer.class, job.tenantId(), job.ownerUserId(), job.itemId());
|
||||
List<String> failureCodes = pageTexts.isEmpty() ? jdbcTemplate.queryForList("""
|
||||
select distinct last_error_code from aihr_personal_ocr_page
|
||||
where tenant_id = ? and owner_user_id = ? and item_id = ?
|
||||
and status = 'FAILED' and last_error_code is not null
|
||||
order by last_error_code
|
||||
""", String.class, job.tenantId(), job.ownerUserId(), job.itemId()) : List.of();
|
||||
transactions.execute(status -> {
|
||||
if (pageTexts.isEmpty()) {
|
||||
String failureCode = terminalFailureCode(failureCodes);
|
||||
jdbcTemplate.update("""
|
||||
update aihr_personal_item
|
||||
set status = 'FAILED', error_code = 'PERSONAL_OCR_EMPTY',
|
||||
error_message = '扫描PDF未识别到可用文字', update_time = now()
|
||||
set status = 'FAILED', error_code = ?, error_message = ?, update_time = now()
|
||||
where tenant_id = ? and owner_user_id = ? and id = ? and status = 'PARSING'
|
||||
""", job.tenantId(), job.ownerUserId(), job.itemId());
|
||||
""", failureCode, terminalFailureMessage(failureCode),
|
||||
job.tenantId(), job.ownerUserId(), job.itemId());
|
||||
} else {
|
||||
publishFragments(job, String.join("\n\n", pageTexts), failedPages);
|
||||
}
|
||||
@@ -374,6 +381,26 @@ public class PersonalPdfOcrService {
|
||||
});
|
||||
}
|
||||
|
||||
private static String terminalFailureCode(List<String> failureCodes) {
|
||||
if (failureCodes.size() == 1 && !"PERSONAL_OCR_PAGE_EMPTY".equals(failureCodes.get(0))) {
|
||||
return failureCodes.get(0);
|
||||
}
|
||||
if (failureCodes.stream().anyMatch(code -> code != null && !"PERSONAL_OCR_PAGE_EMPTY".equals(code))) {
|
||||
return "PERSONAL_OCR_UNAVAILABLE";
|
||||
}
|
||||
return "PERSONAL_OCR_EMPTY";
|
||||
}
|
||||
|
||||
private static String terminalFailureMessage(String failureCode) {
|
||||
return switch (failureCode) {
|
||||
case "PERSONAL_OCR_MODEL_UNAVAILABLE" -> "未配置支持图片的视觉模型，请配置后重试失败页";
|
||||
case "PERSONAL_OCR_RUNTIME_DISABLED" -> "扫描PDF识别服务已关闭，请启用后重试失败页";
|
||||
case "PERSONAL_OCR_PROVIDER_FAILED", "PERSONAL_OCR_RESPONSE_INVALID", "PERSONAL_OCR_UNAVAILABLE" ->
|
||||
"扫描PDF识别服务暂不可用，请稍后重试失败页";
|
||||
default -> "扫描PDF未识别到可用文字";
|
||||
};
|
||||
}
|
||||
|
||||
private void publishFragments(OcrJob job, String text, List<Integer> failedPages) {
|
||||
List<String> chunks = new ParsedDocument(text, "application/pdf", Map.of()).chunks(800, 120);
|
||||
jdbcTemplate.update("""
|
||||
|
||||
+2
-1
@@ -44,7 +44,8 @@ class PersonalIngestionWorkerTest {
|
||||
when(jdbc.queryForList(contains("status = 'QUEUED'"))).thenReturn(List.of(item()));
|
||||
when(jdbc.update(contains("status = 'PARSING'"), eq("000000"), eq(101L), eq(9L), eq(0))).thenReturn(1);
|
||||
when(parser.parse(eq("sample.pdf"), eq("text/plain"), any(byte[].class)))
|
||||
.thenReturn(new ParsedDocument("", "application/pdf", Map.of()));
|
||||
.thenThrow(new KnowledgeDocumentParser.ParseException(
|
||||
KnowledgeDocumentParser.Failure.EMPTY, "document contains no text"));
|
||||
AtomicReference<org.dromara.aihr.personal.service.PersonalPdfOcrService.OcrItem> handedOff =
|
||||
new AtomicReference<>();
|
||||
PersonalIngestionWorker worker = PersonalIngestionWorker.forTestWithPdfOcr(
|
||||
|
||||
+28
@@ -165,6 +165,34 @@ class PersonalPdfOcrServiceTest {
|
||||
eq("000000"), eq(101L), eq(9L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void allPagesRejectedByProviderExposeModelFailureInsteadOfEmptyDocument() {
|
||||
JdbcTemplate jdbc = mock(JdbcTemplate.class);
|
||||
Map<String, Object> job = Map.ofEntries(
|
||||
Map.entry("id", 5L), Map.entry("tenant_id", "000000"), Map.entry("owner_user_id", 101L),
|
||||
Map.entry("item_id", 9L), Map.entry("space_id", 7L), Map.entry("oss_id", 88L),
|
||||
Map.entry("tags_json", "[]"), Map.entry("total_pages", 8), Map.entry("next_page", 9),
|
||||
Map.entry("status", "RETRY"));
|
||||
when(jdbc.queryForList(contains("from aihr_personal_ocr_job"))).thenReturn(List.of(job));
|
||||
when(jdbc.update(contains("set status = 'RUNNING'"), eq(5L))).thenReturn(1);
|
||||
when(jdbc.queryForList(contains("attempt_count < 3"), eq(Integer.class),
|
||||
eq("000000"), eq(101L), eq(9L), eq(20))).thenReturn(List.of());
|
||||
when(jdbc.queryForList(contains("select ocr_text"), eq(String.class),
|
||||
eq("000000"), eq(101L), eq(9L))).thenReturn(List.of());
|
||||
when(jdbc.queryForList(contains("select page_number"), eq(Integer.class),
|
||||
eq("000000"), eq(101L), eq(9L))).thenReturn(List.of(1, 2, 3, 4, 5, 6, 7, 8));
|
||||
when(jdbc.queryForList(contains("select distinct last_error_code"), eq(String.class),
|
||||
eq("000000"), eq(101L), eq(9L))).thenReturn(List.of("PERSONAL_OCR_PROVIDER_FAILED"));
|
||||
PersonalPdfOcrService service = PersonalPdfOcrService.forTest(
|
||||
jdbc, mock(PersonalPdfPageRenderer.class), mock(PersonalVisionOcrService.class),
|
||||
immediateTransactions(), ignored -> new byte[] {1});
|
||||
|
||||
assertTrue(service.processNextBatch());
|
||||
|
||||
verify(jdbc).update(contains("set status = 'FAILED', error_code = ?"),
|
||||
eq("PERSONAL_OCR_PROVIDER_FAILED"), any(String.class), eq("000000"), eq(101L), eq(9L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void recoversExpiredRunningLease() {
|
||||
JdbcTemplate jdbc = mock(JdbcTemplate.class);
|
||||
|
||||
Reference in New Issue
Block a user