fix(knowledge): tolerate non-archive detection failures
This commit is contained in:
+19
@@ -1,6 +1,7 @@
|
|||||||
package org.dromara.aihr.knowledge.parse;
|
package org.dromara.aihr.knowledge.parse;
|
||||||
|
|
||||||
import org.apache.tika.exception.WriteLimitReachedException;
|
import org.apache.tika.exception.WriteLimitReachedException;
|
||||||
|
import org.apache.tika.detect.Detector;
|
||||||
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
|
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
|
||||||
import org.apache.tika.io.BoundedInputStream;
|
import org.apache.tika.io.BoundedInputStream;
|
||||||
import org.apache.tika.io.TemporaryResources;
|
import org.apache.tika.io.TemporaryResources;
|
||||||
@@ -8,6 +9,7 @@ import org.apache.tika.io.TikaInputStream;
|
|||||||
import org.apache.tika.metadata.Metadata;
|
import org.apache.tika.metadata.Metadata;
|
||||||
import org.apache.tika.metadata.TikaCoreProperties;
|
import org.apache.tika.metadata.TikaCoreProperties;
|
||||||
import org.apache.tika.mime.MediaType;
|
import org.apache.tika.mime.MediaType;
|
||||||
|
import org.apache.tika.mime.MimeTypes;
|
||||||
import org.apache.tika.parser.AutoDetectParser;
|
import org.apache.tika.parser.AutoDetectParser;
|
||||||
import org.apache.tika.parser.ParseContext;
|
import org.apache.tika.parser.ParseContext;
|
||||||
import org.apache.tika.parser.Parser;
|
import org.apache.tika.parser.Parser;
|
||||||
@@ -60,6 +62,8 @@ public class TikaKnowledgeDocumentParser implements KnowledgeDocumentParser {
|
|||||||
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, fileName.trim());
|
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, fileName.trim());
|
||||||
}
|
}
|
||||||
AutoDetectParser parser = new AutoDetectParser();
|
AutoDetectParser parser = new AutoDetectParser();
|
||||||
|
Detector detector = parser.getDetector();
|
||||||
|
parser.setDetector((stream, currentMetadata) -> safeDetect(detector, stream, currentMetadata));
|
||||||
BodyContentHandler handler = new BodyContentHandler(maxExpandedChars + 1);
|
BodyContentHandler handler = new BodyContentHandler(maxExpandedChars + 1);
|
||||||
BoundedInputStream bounded = new BoundedInputStream(MAX_INPUT_BYTES + 1, input);
|
BoundedInputStream bounded = new BoundedInputStream(MAX_INPUT_BYTES + 1, input);
|
||||||
try (TemporaryResources temporaryResources = new TemporaryResources();
|
try (TemporaryResources temporaryResources = new TemporaryResources();
|
||||||
@@ -91,6 +95,21 @@ public class TikaKnowledgeDocumentParser implements KnowledgeDocumentParser {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static MediaType safeDetect(Detector detector, InputStream input, Metadata metadata) throws IOException {
|
||||||
|
input.mark(Integer.MAX_VALUE);
|
||||||
|
try {
|
||||||
|
return detector.detect(input, metadata);
|
||||||
|
} catch (Exception exception) {
|
||||||
|
if (!(exception instanceof org.apache.commons.compress.archivers.ArchiveException)) {
|
||||||
|
if (exception instanceof IOException ioException) throw ioException;
|
||||||
|
if (exception instanceof RuntimeException runtimeException) throw runtimeException;
|
||||||
|
throw new IOException("document type detection failed", exception);
|
||||||
|
}
|
||||||
|
input.reset();
|
||||||
|
return MimeTypes.getDefaultMimeTypes().detect(input, metadata);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private ParsedDocument parsedDocument(BodyContentHandler handler, Metadata metadata, String mimeType) {
|
private ParsedDocument parsedDocument(BodyContentHandler handler, Metadata metadata, String mimeType) {
|
||||||
String text = handler.toString().trim();
|
String text = handler.toString().trim();
|
||||||
if (text.isEmpty()) {
|
if (text.isEmpty()) {
|
||||||
|
|||||||
Reference in New Issue
Block a user