作成されたインデックスと実ファイルに差分がある

HAHA · June 29, 2023, 12:36pm

お忙しいところお手数ですが質問させてください。
クロールによって作成されたインデックスと実ファイルに差分がありました。
fess-crawler.logおよび障害URLを確認したところ以下の4点のエラーメッセージを確認しております。

特に１）、２）の原因特定ができていないため、
ご知見ございましたらご教授いただきたく存じます。

１）java.lang.ArrayIndexOutOfBoundsException

2023-06-21 11:20:12,956 [Crawler-20230621111833-1-5] WARN Unexpected error processing command, ignoring and continuing. Command: org.apache.poi.hdgf.chunks.Chunk$Command@476c7347
java.lang.ArrayIndexOutOfBoundsException: Index 23 out of bounds for length 20
at org.apache.poi.util.LittleEndian.getLong(LittleEndian.java:155) ~[poi-5.2.3.jar:5.2.3]
at org.apache.poi.util.LittleEndian.getDouble(LittleEndian.java:74) ~[poi-5.2.3.jar:5.2.3]
at org.apache.poi.hdgf.chunks.Chunk.processCommands(Chunk.java:199) ~[poi-scratchpad-5.2.3.jar:5.2.3]
at org.apache.poi.hdgf.chunks.ChunkFactory.createChunk(ChunkFactory.java:207) ~[poi-scratchpad-5.2.3.jar:5.2.3]
at org.apache.poi.hdgf.streams.ChunkStream.findChunks(ChunkStream.java:66) ~[poi-scratchpad-5.2.3.jar:5.2.3]
at org.apache.poi.hdgf.streams.PointerContainingStream.findChildren(PointerContainingStream.java:70) ~[poi-scratchpad-5.2.3.jar:5.2.3]
at org.apache.poi.hdgf.streams.PointerContainingStream.findChildren(PointerContainingStream.java:77) ~[poi-scratchpad-5.2.3.jar:5.2.3]
at org.apache.poi.hdgf.HDGFDiagram.(HDGFDiagram.java:89) ~[poi-scratchpad-5.2.3.jar:5.2.3]
at org.apache.poi.hdgf.extractor.VisioTextExtractor.(VisioTextExtractor.java:52) ~[poi-scratchpad-5.2.3.jar:5.2.3]
at org.apache.tika.parser.microsoft.OfficeParser.parse(OfficeParser.java:232) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.microsoft.OfficeParser.parse(OfficeParser.java:175) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.CompositeParser.parse(CompositeParser.java:298) ~[tika-core-2.6.0.jar:2.6.0]
at org.apache.tika.parser.CompositeParser.parse(CompositeParser.java:298) ~[tika-core-2.6.0.jar:2.6.0]
at org.codelibs.fess.crawler.extractor.impl.TikaExtractor$TikaDetectParser.parse(TikaExtractor.java:507) ~[fess-crawler-14.6.1.jar:?]
at org.apache.tika.parser.DelegatingParser.parse(DelegatingParser.java:71) ~[tika-core-2.6.0.jar:2.6.0]
at org.apache.tika.extractor.ParsingEmbeddedDocumentExtractor.parseEmbedded(ParsingEmbeddedDocumentExtractor.java:109) ~[tika-core-2.6.0.jar:2.6.0]
at org.apache.tika.parser.microsoft.ooxml.AbstractOOXMLExtractor.handleEmbeddedFile(AbstractOOXMLExtractor.java:406) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.microsoft.ooxml.AbstractOOXMLExtractor.handleEmbeddedPart(AbstractOOXMLExtractor.java:275) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.microsoft.ooxml.AbstractOOXMLExtractor.handleEmbeddedParts(AbstractOOXMLExtractor.java:217) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.microsoft.ooxml.AbstractOOXMLExtractor.getXHTML(AbstractOOXMLExtractor.java:138) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.microsoft.ooxml.OOXMLExtractorFactory.parse(OOXMLExtractorFactory.java:243) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.microsoft.ooxml.OOXMLParser.parse(OOXMLParser.java:115) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.microsoft.OfficeParser.parse(OfficeParser.java:269) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.microsoft.OfficeParser.parse(OfficeParser.java:175) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.CompositeParser.parse(CompositeParser.java:298) ~[tika-core-2.6.0.jar:2.6.0]
at org.apache.tika.parser.CompositeParser.parse(CompositeParser.java:298) ~[tika-core-2.6.0.jar:2.6.0]
at org.codelibs.fess.crawler.extractor.impl.TikaExtractor$TikaDetectParser.parse(TikaExtractor.java:507) ~[fess-crawler-14.6.1.jar:?]
at org.codelibs.fess.crawler.extractor.impl.TikaExtractor.lambda$getText$0(TikaExtractor.java:194) ~[fess-crawler-14.6.1.jar:?]
at org.codelibs.fess.crawler.extractor.impl.TikaExtractor.getContent(TikaExtractor.java:404) ~[fess-crawler-14.6.1.jar:?]
at org.codelibs.fess.crawler.extractor.impl.TikaExtractor.getText(TikaExtractor.java:183) ~[fess-crawler-14.6.1.jar:?]
at org.codelibs.fess.crawler.transformer.AbstractFessFileTransformer.getExtractData(AbstractFessFileTransformer.java:387) ~[classes/:?]
at org.codelibs.fess.crawler.transformer.AbstractFessFileTransformer.generateData(AbstractFessFileTransformer.java:100) ~[classes/:?]
at org.codelibs.fess.crawler.transformer.AbstractFessFileTransformer.transform(AbstractFessFileTransformer.java:81) ~[classes/:?]
at org.codelibs.fess.crawler.processor.impl.DefaultResponseProcessor.process(DefaultResponseProcessor.java:74) ~[fess-crawler-14.6.1.jar:?]
at org.codelibs.fess.crawler.CrawlerThread.processResponse(CrawlerThread.java:291) ~[fess-crawler-14.6.1.jar:?]
at org.codelibs.fess.crawler.FessCrawlerThread.processResponse(FessCrawlerThread.java:249) ~[classes/:?]
at org.codelibs.fess.crawler.CrawlerThread.run(CrawlerThread.java:162) ~[fess-crawler-14.6.1.jar:?]
at java.lang.Thread.run(Thread.java:833) ~[?:?]

２）java.lang.IllegalArgumentException

2023-06-21 11:20:23,978 [Crawler-20230621111833-1-3] WARN Failed to create chunk at 20885, ignoring rest of data.
java.lang.IllegalArgumentException: Found a chunk with a negative length, which isn’t allowed
at org.apache.poi.hdgf.chunks.ChunkFactory.createChunk(ChunkFactory.java:149) ~[poi-scratchpad-5.2.3.jar:5.2.3]
at org.apache.poi.hdgf.streams.ChunkStream.findChunks(ChunkStream.java:66) ~[poi-scratchpad-5.2.3.jar:5.2.3]
at org.apache.poi.hdgf.streams.PointerContainingStream.findChildren(PointerContainingStream.java:70) ~[poi-scratchpad-5.2.3.jar:5.2.3]
at org.apache.poi.hdgf.streams.PointerContainingStream.findChildren(PointerContainingStream.java:77) ~[poi-scratchpad-5.2.3.jar:5.2.3]
at org.apache.poi.hdgf.HDGFDiagram.(HDGFDiagram.java:89) ~[poi-scratchpad-5.2.3.jar:5.2.3]
at org.apache.poi.hdgf.extractor.VisioTextExtractor.(VisioTextExtractor.java:52) ~[poi-scratchpad-5.2.3.jar:5.2.3]
at org.apache.tika.parser.microsoft.OfficeParser.parse(OfficeParser.java:232) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.microsoft.OfficeParser.parse(OfficeParser.java:175) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.CompositeParser.parse(CompositeParser.java:298) ~[tika-core-2.6.0.jar:2.6.0]
at org.apache.tika.parser.CompositeParser.parse(CompositeParser.java:298) ~[tika-core-2.6.0.jar:2.6.0]
at org.codelibs.fess.crawler.extractor.impl.TikaExtractor$TikaDetectParser.parse(TikaExtractor.java:507) ~[fess-crawler-14.6.1.jar:?]
at org.apache.tika.parser.DelegatingParser.parse(DelegatingParser.java:71) ~[tika-core-2.6.0.jar:2.6.0]
at org.apache.tika.extractor.ParsingEmbeddedDocumentExtractor.parseEmbedded(ParsingEmbeddedDocumentExtractor.java:109) ~[tika-core-2.6.0.jar:2.6.0]
at org.apache.tika.parser.microsoft.ooxml.AbstractOOXMLExtractor.handleEmbeddedFile(AbstractOOXMLExtractor.java:406) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.microsoft.ooxml.AbstractOOXMLExtractor.handleEmbeddedPart(AbstractOOXMLExtractor.java:275) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.microsoft.ooxml.AbstractOOXMLExtractor.handleEmbeddedParts(AbstractOOXMLExtractor.java:217) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.microsoft.ooxml.AbstractOOXMLExtractor.getXHTML(AbstractOOXMLExtractor.java:138) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.microsoft.ooxml.OOXMLExtractorFactory.parse(OOXMLExtractorFactory.java:243) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.microsoft.ooxml.OOXMLParser.parse(OOXMLParser.java:115) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.microsoft.OfficeParser.parse(OfficeParser.java:269) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.microsoft.OfficeParser.parse(OfficeParser.java:175) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.CompositeParser.parse(CompositeParser.java:298) ~[tika-core-2.6.0.jar:2.6.0]
at org.apache.tika.parser.CompositeParser.parse(CompositeParser.java:298) ~[tika-core-2.6.0.jar:2.6.0]
at org.codelibs.fess.crawler.extractor.impl.TikaExtractor$TikaDetectParser.parse(TikaExtractor.java:507) ~[fess-crawler-14.6.1.jar:?]
at org.codelibs.fess.crawler.extractor.impl.TikaExtractor.lambda$getText$0(TikaExtractor.java:194) ~[fess-crawler-14.6.1.jar:?]
at org.codelibs.fess.crawler.extractor.impl.TikaExtractor.getContent(TikaExtractor.java:404) ~[fess-crawler-14.6.1.jar:?]
at org.codelibs.fess.crawler.extractor.impl.TikaExtractor.getText(TikaExtractor.java:183) ~[fess-crawler-14.6.1.jar:?]
at org.codelibs.fess.crawler.transformer.AbstractFessFileTransformer.getExtractData(AbstractFessFileTransformer.java:387) ~[classes/:?]
at org.codelibs.fess.crawler.transformer.AbstractFessFileTransformer.generateData(AbstractFessFileTransformer.java:100) ~[classes/:?]
at org.codelibs.fess.crawler.transformer.AbstractFessFileTransformer.transform(AbstractFessFileTransformer.java:81) ~[classes/:?]
at org.codelibs.fess.crawler.processor.impl.DefaultResponseProcessor.process(DefaultResponseProcessor.java:74) ~[fess-crawler-14.6.1.jar:?]
at org.codelibs.fess.crawler.CrawlerThread.processResponse(CrawlerThread.java:291) ~[fess-crawler-14.6.1.jar:?]
at org.codelibs.fess.crawler.FessCrawlerThread.processResponse(FessCrawlerThread.java:249) ~[classes/:?]
at org.codelibs.fess.crawler.CrawlerThread.run(CrawlerThread.java:162) ~[fess-crawler-14.6.1.jar:?]

３）java.net.URISyntaxException: Illegal character in path at
→ファイル名にスペースが含まれると発生する。ファイル名を修正する対応予定。
他にfess側で対応すること可能なのでしょうか。

４）MaxLengthExceededException
→ファイルサイズの上限を超えた際に発生するエラー。デフォルトは10MB。
設定上の上限値はありますでしょうか。

shinsuke · June 29, 2023, 1:40pm

Apache POIが処理する上で、出力しているただのWARNログのように見えます。POIやPDFBoxはINFO的な情報もWARNで出したりします。
URISyntaxExceptionはその情報だけでは何とも言えません。
クロール関連の設定を参照してください。

HAHA · July 4, 2023, 10:41am

ご回答ありがとうございます。

ご指摘の内容を受け、クロール設定を見直したところインデックスが作成できました（ファイルサイズの都合上一部まだですが。。）
そのうえで気になった部分がありましたのでご教示いただけますと幸いです。
お忙しいところ申し訳ございませんが、宜しくお願いいたします。

１）検索ワードに"・“や”_"といった記号が入れると、検索に引っかからない。
（例：ファイル名が”AAA・BBB.doc”だった場合
AAA・BBB→NG、BBB→OK）
こちら設定等で解消すること可能なのでしょうか。

２）クロールログで気になった部分がございましたので内容が判断できず、ご知見あればご教授いただきたく存じます。

2023-07-04 15:21:41,018 [Crawler-20230704152124-1-3] WARN Cannot convert file:///ファイルパス in a valid relationship URI-> dummy-URI used
java.net.URISyntaxException: Illegal character in path at index 81: file://///ファイルパス
at java.net.URI$Parser.fail(URI.java:2976) ~[?:?]
at java.net.URI$Parser.checkChars(URI.java:3147) ~[?:?]
at java.net.URI$Parser.parseHierarchical(URI.java:3229) ~[?:?]
at java.net.URI$Parser.parse(URI.java:3177) ~[?:?]
at java.net.URI.(URI.java:623) ~[?:?]
at org.apache.poi.openxml4j.opc.PackagingURIHelper.toURI(PackagingURIHelper.java:723) ~[poi-ooxml-5.2.3.jar:5.2.3]
at org.apache.poi.openxml4j.opc.PackageRelationshipCollection.parseRelationshipsPart(PackageRelationshipCollection.java:358) ~[poi-ooxml-5.2.3.jar:5.2.3]
at org.apache.poi.openxml4j.opc.PackageRelationshipCollection.(PackageRelationshipCollection.java:160) ~[poi-ooxml-5.2.3.jar:5.2.3]
at org.apache.poi.openxml4j.opc.PackageRelationshipCollection.(PackageRelationshipCollection.java:130) ~[poi-ooxml-5.2.3.jar:5.2.3]
at org.apache.poi.openxml4j.opc.PackagePart.loadRelationships(PackagePart.java:565) ~[poi-ooxml-5.2.3.jar:5.2.3]
at org.apache.poi.openxml4j.opc.OPCPackage.getParts(OPCPackage.java:751) ~[poi-ooxml-5.2.3.jar:5.2.3]
at org.apache.poi.openxml4j.opc.OPCPackage.open(OPCPackage.java:322) ~[poi-ooxml-5.2.3.jar:5.2.3]
at org.apache.tika.parser.microsoft.ooxml.OOXMLExtractorFactory.parse(OOXMLExtractorFactory.java:123) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.microsoft.ooxml.OOXMLParser.parse(OOXMLParser.java:115) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.microsoft.OfficeParser.parse(OfficeParser.java:269) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.microsoft.OfficeParser.parse(OfficeParser.java:175) ~[tika-parser-microsoft-module-2.6.0.jar:2.6.0]
at org.apache.tika.parser.CompositeParser.parse(CompositeParser.java:298) ~[tika-core-2.6.0.jar:2.6.0]
at org.apache.tika.parser.CompositeParser.parse(CompositeParser.java:298) ~[tika-core-2.6.0.jar:2.6.0]
at org.codelibs.fess.crawler.extractor.impl.TikaExtractor$TikaDetectParser.parse(TikaExtractor.java:507) ~[fess-crawler-14.6.1.jar:?]
at org.codelibs.fess.crawler.extractor.impl.TikaExtractor.lambda$getText$0(TikaExtractor.java:194) ~[fess-crawler-14.6.1.jar:?]
at org.codelibs.fess.crawler.extractor.impl.TikaExtractor.getContent(TikaExtractor.java:404) ~[fess-crawler-14.6.1.jar:?]
at org.codelibs.fess.crawler.extractor.impl.TikaExtractor.getText(TikaExtractor.java:183) ~[fess-crawler-14.6.1.jar:?]
at org.codelibs.fess.crawler.transformer.AbstractFessFileTransformer.getExtractData(AbstractFessFileTransformer.java:387) ~[classes/:?]
at org.codelibs.fess.crawler.transformer.AbstractFessFileTransformer.generateData(AbstractFessFileTransformer.java:100) ~[classes/:?]
at org.codelibs.fess.crawler.transformer.AbstractFessFileTransformer.transform(AbstractFessFileTransformer.java:81) ~[classes/:?]
at org.codelibs.fess.crawler.processor.impl.DefaultResponseProcessor.process(DefaultResponseProcessor.java:74) ~[fess-crawler-14.6.1.jar:?]
at org.codelibs.fess.crawler.CrawlerThread.processResponse(CrawlerThread.java:291) ~[fess-crawler-14.6.1.jar:?]
at org.codelibs.fess.crawler.FessCrawlerThread.processResponse(FessCrawlerThread.java:249) ~[classes/:?]
at org.codelibs.fess.crawler.CrawlerThread.run(CrawlerThread.java:162) ~[fess-crawler-14.6.1.jar:?]
at java.lang.Thread.run(Thread.java:833) ~[?:?]
2023-07-04 15:21:41,258 [IndexUpdater] INFO Processing 4/11 docs (Doc:{access 0ms}, Mem:{used 150MB, heap 354MB, max 3GB})
2023-07-04 15:21:41,477 [IndexUpdater] INFO Processing 0/7 docs (Doc:{access 16ms, cleanup 78ms}, Mem:{used 153MB, heap 354MB, max 3GB})

shinsuke · July 4, 2023, 11:52am

1は、テキスト解析に関する個別要件については、fess.jsonを修正して、インデックスを再作成してください。
2は、POIの問題だと思います。