We had been running FESS successfully with the following docker-compose.yml config until FESS 13.10.4:
# docker-compose.yml
version: "3"
services:
fess:
image: ghcr.io/codelibs/fess:13.10.4
restart: always
ports:
- "8080:8080"
depends_on:
- elasticsearch
environment:
- RUN_ELASTICSEARCH=false
- "ES_HTTP_URL=http://elasticsearch:9200"
- "ES_TRANSPORT_URL=elasticsearch:9300"
ulimits:
nproc: 65535
nofile:
soft: 65536
hard: 65536
memlock:
soft: -1
hard: -1
volumes:
- ./fess/log:/var/log/fess
- ./fess/data:/var/lib/fess
networks:
- esnet
elasticsearch:
image: ghcr.io/codelibs/fess:13.10.4
restart: always
environment:
- RUN_FESS=false
- "ES_JAVA_OPTS=\"-Xms4g -Xmx4g\""
ulimits:
nproc: 65535
nofile:
soft: 65536
hard: 65536
memlock:
soft: -1
hard: -1
volumes:
- ./es/data:/var/lib/elasticsearch
- ./es/log:/var/log/elasticsearch
networks:
- esnet
networks:
esnet:
driver: bridge
To update FESS, we would usually just change the image versions and run docker-compose up -d. We would not lose any data.
However, this did no longer work when 13.11 was released, as Elasticsearch seemed to have been silently removed from the ghcr.io/codelibs/fess image, and a ghcr.io/codelibs/fess-elasticsearch-oss popped up at the same time.
Unfortunately, just exchanging the elasticsearch service’s image for ghcr.io/codelibs/fess-elasticsearch-oss:7.10.2` (as seen here) does not work either:
# docker-compose.yml (new)
version: "3"
services:
fess:
image: ghcr.io/codelibs/fess:13.10.4
restart: always
ports:
- "8080:8080"
depends_on:
- elasticsearch
environment:
- RUN_ELASTICSEARCH=false
- "ES_HTTP_URL=http://elasticsearch:9200"
- "FESS_DICTIONARY_PATH=/usr/share/elasticsearch/config/dictionary"
ulimits:
nproc: 65535
nofile:
soft: 65536
hard: 65536
memlock:
soft: -1
hard: -1
volumes:
- ./fess/log:/var/log/fess
- ./fess/data:/var/lib/fess
networks:
- esnet
elasticsearch:
image: ghcr.io/codelibs/fess-elasticsearch-oss:7.10.2
restart: always
environment:
- node.name=elasticsearch
- discovery.seed_hosts=elasticsearch
- cluster.initial_master_nodes=elasticsearch
- cluster.name=elasticsearch
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms4g -Xmx4g"
- "FESS_DICTIONARY_PATH=/usr/share/elasticsearch/config/dictionary"
ulimits:
nproc: 65535
nofile:
soft: 65536
hard: 65536
memlock:
soft: -1
hard: -1
volumes:
- ./es/data:/usr/share/elasticsearch/data
- ./es/dictionary:/usr/share/elasticsearch/config/dictionary
- ./es/log:/var/log/elasticsearch
networks:
- esnet
networks:
esnet:
driver: bridge
The new image does not pick up the existing ES data, causing empty search results and making it impossible to login with our admin credentials, if the services run at all (see errors below).
Therefore my question is: What is the recommended way to migrate from fess:13.10.4 (for both FESS and ES) to fess:13.15.2 and fess-elasticsearch:7.15.2, without losing any data?
PS: Fess service fails with the following error:
fess_1 |
fess_1 | ==> /var/log/fess/fess.log <==
fess_1 | 2021-12-13 11:08:49,486 [main] ERROR Failed to initialize Lasta Di.
fess_1 | org.lastaflute.di.exception.ContainerInitFailureException: Look! Read the message below.
fess_1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
fess_1 | Failed to initialize the container.
fess_1 |
fess_1 | [Path]
fess_1 | app.xml
fess_1 |
fess_1 | [Namespace]
fess_1 | null
fess_1 | * * * * * * * * * */
fess_1 | at org.lastaflute.di.core.meta.impl.LaContainerImpl.throwContainerInitFailureException(LaContainerImpl.java:463) ~[lasta-di-0.8.3.jar:?]
fess_1 | at org.lastaflute.di.core.meta.impl.LaContainerImpl.init(LaContainerImpl.java:415) ~[lasta-di-0.8.3.jar:?]
fess_1 | at org.lastaflute.di.core.factory.SingletonLaContainerFactory.init(SingletonLaContainerFactory.java:67) ~[lasta-di-0.8.3.jar:?]
fess_1 | at org.lastaflute.web.container.WebLastaContainerInitializer.doInitContainer(WebLastaContainerInitializer.java:81) ~[lastaflute-1.1.8.jar:?]
fess_1 | at org.lastaflute.web.container.WebLastaContainerInitializer.initialize(WebLastaContainerInitializer.java:45) ~[lastaflute-1.1.8.jar:?]
fess_1 | at org.lastaflute.web.servlet.filter.LastaPrepareFilter.initializeContainer(LastaPrepareFilter.java:137) [lastaflute-1.1.8.jar:?]
fess_1 | at org.lastaflute.web.servlet.filter.LastaPrepareFilter.init(LastaPrepareFilter.java:94) [lastaflute-1.1.8.jar:?]
fess_1 | at org.apache.catalina.core.ApplicationFilterConfig.initFilter(ApplicationFilterConfig.java:270) [classes/:?]
fess_1 | at org.apache.catalina.core.ApplicationFilterConfig.getFilter(ApplicationFilterConfig.java:251) [classes/:?]
fess_1 | at org.apache.catalina.core.ApplicationFilterConfig.<init>(ApplicationFilterConfig.java:102) [classes/:?]
fess_1 | at org.apache.catalina.core.StandardContext.filterStart(StandardContext.java:4566) [classes/:?]
fess_1 | at org.apache.catalina.core.StandardContext.startInternal(StandardContext.java:5203) [classes/:?]
fess_1 | at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:183) [classes/:?]
fess_1 | at org.apache.catalina.core.ContainerBase$StartChild.call(ContainerBase.java:1384) [classes/:?]
fess_1 | at org.apache.catalina.core.ContainerBase$StartChild.call(ContainerBase.java:1374) [classes/:?]
fess_1 | at java.util.concurrent.FutureTask.run(Unknown Source) [?:?]
fess_1 | at org.apache.tomcat.util.threads.InlineExecutorService.execute(InlineExecutorService.java:75) [classes/:?]
fess_1 | at java.util.concurrent.AbstractExecutorService.submit(Unknown Source) [?:?]
fess_1 | at org.apache.catalina.core.ContainerBase.startInternal(ContainerBase.java:909) [classes/:?]
fess_1 | at org.apache.catalina.core.StandardHost.startInternal(StandardHost.java:843) [classes/:?]
fess_1 | at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:183) [classes/:?]
fess_1 | at org.apache.catalina.core.ContainerBase$StartChild.call(ContainerBase.java:1384) [classes/:?]
fess_1 | at org.apache.catalina.core.ContainerBase$StartChild.call(ContainerBase.java:1374) [classes/:?]
fess_1 | at java.util.concurrent.FutureTask.run(Unknown Source) [?:?]
fess_1 | at org.apache.tomcat.util.threads.InlineExecutorService.execute(InlineExecutorService.java:75) [classes/:?]
fess_1 | at java.util.concurrent.AbstractExecutorService.submit(Unknown Source) [?:?]
fess_1 | at org.apache.catalina.core.ContainerBase.startInternal(ContainerBase.java:909) [classes/:?]
fess_1 | at org.apache.catalina.core.StandardEngine.startInternal(StandardEngine.java:262) [classes/:?]
fess_1 | at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:183) [classes/:?]
fess_1 | at org.apache.catalina.core.StandardService.startInternal(StandardService.java:434) [classes/:?]
fess_1 | at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:183) [classes/:?]
fess_1 | at org.apache.catalina.core.StandardServer.startInternal(StandardServer.java:930) [classes/:?]
fess_1 | at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:183) [classes/:?]
fess_1 | at org.apache.catalina.startup.Tomcat.start(Tomcat.java:486) [classes/:?]
fess_1 | at org.dbflute.tomcat.TomcatBoot.startServer(TomcatBoot.java:713) [classes/:?]
fess_1 | at org.dbflute.tomcat.TomcatBoot.go(TomcatBoot.java:402) [classes/:?]
fess_1 | at org.dbflute.tomcat.TomcatBoot.bootAwait(TomcatBoot.java:347) [classes/:?]
fess_1 | at org.codelibs.fess.FessBoot.main(FessBoot.java:96) [classes/:?]
fess_1 | Caused by: org.lastaflute.di.core.exception.IllegalMethodRuntimeException: [ESSR0060]Can not configure method(init) of org.codelibs.fess.helper.SuggestHelper, because ElasticsearchStatusException[Elasticsearch exception [type=search_phase_execution_exception, reason=all shards failed]]
fess_1 | at org.lastaflute.di.core.assembler.AbstractMethodAssembler.invoke(AbstractMethodAssembler.java:61) ~[lasta-di-0.8.3.jar:?]
fess_1 | at org.lastaflute.di.core.assembler.DefaultInitMethodAssembler.assemble(DefaultInitMethodAssembler.java:40) ~[lasta-di-0.8.3.jar:?]
fess_1 | at org.lastaflute.di.core.deployer.SingletonComponentDeployer.assemble(SingletonComponentDeployer.java:57) ~[lasta-di-0.8.3.jar:?]
fess_1 | at org.lastaflute.di.core.deployer.SingletonComponentDeployer.deploy(SingletonComponentDeployer.java:41) ~[lasta-di-0.8.3.jar:?]
fess_1 | at org.lastaflute.di.core.deployer.SingletonComponentDeployer.init(SingletonComponentDeployer.java:35) ~[lasta-di-0.8.3.jar:?]
fess_1 | at org.lastaflute.di.core.meta.impl.ComponentDefImpl.init(ComponentDefImpl.java:85) ~[lasta-di-0.8.3.jar:?]
fess_1 | at org.lastaflute.di.core.meta.impl.LaContainerImpl.doInit(LaContainerImpl.java:434) ~[lasta-di-0.8.3.jar:?]
fess_1 | at org.lastaflute.di.core.meta.impl.LaContainerImpl.init(LaContainerImpl.java:413) ~[lasta-di-0.8.3.jar:?]
fess_1 | ... 36 more
fess_1 | Caused by: org.elasticsearch.ElasticsearchStatusException: Elasticsearch exception [type=search_phase_execution_exception, reason=all shards failed]
fess_1 | at org.elasticsearch.rest.BytesRestResponse.errorFromXContent(BytesRestResponse.java:187) ~[elasticsearch-7.10.1.jar:7.10.1]
fess_1 | at org.codelibs.elasticsearch.client.action.HttpAction.toElasticsearchException(HttpAction.java:138) ~[elasticsearch-httpclient-7.10.0.jar:?]
fess_1 | at org.codelibs.elasticsearch.client.action.HttpSearchAction.lambda$execute$0(HttpSearchAction.java:48) ~[elasticsearch-httpclient-7.10.0.jar:?]
fess_1 | at org.codelibs.curl.CurlRequest.lambda$execute$4(CurlRequest.java:220) ~[curl4j-1.2.4.jar:?]
fess_1 | at org.codelibs.curl.CurlRequest.lambda$connect$3(CurlRequest.java:199) ~[curl4j-1.2.4.jar:?]
fess_1 | at java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(Unknown Source) ~[?:?]
fess_1 | at java.util.concurrent.ForkJoinTask.doExec(Unknown Source) ~[?:?]
fess_1 | at java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(Unknown Source) ~[?:?]
fess_1 | at java.util.concurrent.ForkJoinPool.scan(Unknown Source) ~[?:?]
fess_1 | at java.util.concurrent.ForkJoinPool.runWorker(Unknown Source) ~[?:?]
fess_1 | at java.util.concurrent.ForkJoinWorkerThread.run(Unknown Source) ~[?:?]
fess_1 | Suppressed: org.elasticsearch.ElasticsearchException: hits is null.
fess_1 | at org.codelibs.elasticsearch.client.action.HttpSearchAction.lambda$execute$0(HttpSearchAction.java:48) ~[elasticsearch-httpclient-7.10.0.jar:?]
fess_1 | at org.codelibs.curl.CurlRequest.lambda$execute$4(CurlRequest.java:220) ~[curl4j-1.2.4.jar:?]
fess_1 | at org.codelibs.curl.CurlRequest.lambda$connect$3(CurlRequest.java:199) ~[curl4j-1.2.4.jar:?]
fess_1 | at java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(Unknown Source) ~[?:?]
fess_1 | at java.util.concurrent.ForkJoinTask.doExec(Unknown Source) ~[?:?]
fess_1 | at java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(Unknown Source) ~[?:?]
fess_1 | at java.util.concurrent.ForkJoinPool.scan(Unknown Source) ~[?:?]
fess_1 | at java.util.concurrent.ForkJoinPool.runWorker(Unknown Source) ~[?:?]
fess_1 | at java.util.concurrent.ForkJoinWorkerThread.run(Unknown Source) ~[?:?]
fess_1 | Suppressed: org.codelibs.elasticsearch.client.action.HttpAction$CurlResponseException: {"error":{"root_cause":[],"type":"search_phase_execution_exception","reason":"all shards failed","phase":"query","grouped":true,"failed_shards":[]},"status":503}
Elasticsearch fails with a lot of these errors:
elasticsearch_1 | {"type": "server", "timestamp": "2021-12-13T11:08:45,575Z", "level": "WARN", "component": "o.c.e.c.s.ConfigSyncService", "cluster.name": "elasticsearch", "node.name": "elasticsearch", "message": "Failed to create /usr/share/elasticsearch/config/dictionary/tr", "cluster.uuid": "_3m_7YwPRGieuJDiWha8Nw", "node.id": "pnA-MHoASDahoW7Z2uZrbQ" }
elasticsearch_1 | {"type": "server", "timestamp": "2021-12-13T11:08:45,578Z", "level": "WARN", "component": "o.c.e.c.s.ConfigSyncService", "cluster.name": "elasticsearch", "node.name": "elasticsearch", "message": "Failed to update tr/stemmer_override.txt", "cluster.uuid": "_3m_7YwPRGieuJDiWha8Nw", "node.id": "pnA-MHoASDahoW7Z2uZrbQ" ,
elasticsearch_1 | "stacktrace": ["java.io.FileNotFoundException: /usr/share/elasticsearch/config/dictionary/tr/stemmer_override.txt (No such file or directory)",
elasticsearch_1 | "at java.io.FileOutputStream.open0(Native Method) ~[?:?]",
elasticsearch_1 | "at java.io.FileOutputStream.open(FileOutputStream.java:291) ~[?:?]",
elasticsearch_1 | "at java.io.FileOutputStream.<init>(FileOutputStream.java:234) ~[?:?]",
elasticsearch_1 | "at java.io.FileOutputStream.<init>(FileOutputStream.java:123) ~[?:?]",
elasticsearch_1 | "at org.codelibs.elasticsearch.configsync.service.ConfigSyncService.decodeToFile(ConfigSyncService.java:701) ~[elasticsearch-configsync-7.10.0.jar:?]",
elasticsearch_1 | "at org.codelibs.elasticsearch.configsync.service.ConfigSyncService.lambda$updateConfigFile$23(ConfigSyncService.java:510) ~[elasticsearch-configsync-7.10.0.jar:?]",
elasticsearch_1 | "at java.security.AccessController.doPrivileged(AccessController.java:312) ~[?:?]",
elasticsearch_1 | "at org.codelibs.elasticsearch.configsync.service.ConfigSyncService.updateConfigFile(ConfigSyncService.java:495) [elasticsearch-configsync-7.10.0.jar:?]",
elasticsearch_1 | "at org.codelibs.elasticsearch.configsync.service.ConfigSyncService.access$1100(ConfigSyncService.java:82) [elasticsearch-configsync-7.10.0.jar:?]",
elasticsearch_1 | "at org.codelibs.elasticsearch.configsync.service.ConfigSyncService$ConfigFileWriter.onResponse(ConfigSyncService.java:604) [elasticsearch-configsync-7.10.0.jar:?]",
elasticsearch_1 | "at org.codelibs.elasticsearch.configsync.service.ConfigSyncService$ConfigFileWriter.onResponse(ConfigSyncService.java:566) [elasticsearch-configsync-7.10.0.jar:?]",
elasticsearch_1 | "at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:89) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:83) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.action.search.SearchScrollAsyncAction.sendResponse(SearchScrollAsyncAction.java:229) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.action.search.SearchScrollAsyncAction$2.run(SearchScrollAsyncAction.java:213) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.action.search.SearchScrollAsyncAction$1.innerOnResponse(SearchScrollAsyncAction.java:167) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:45) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.action.search.SearchActionListener.onResponse(SearchActionListener.java:29) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.action.ActionListenerResponseHandler.handleResponse(ActionListenerResponseHandler.java:54) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleResponse(SearchTransportService.java:399) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.transport.TransportService$6.handleResponse(TransportService.java:634) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleResponse(TransportService.java:1171) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.transport.TransportService$DirectResponseChannel.processResponse(TransportService.java:1249) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.transport.TransportService$DirectResponseChannel.sendResponse(TransportService.java:1229) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.transport.TaskTransportChannel.sendResponse(TaskTransportChannel.java:52) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.action.support.ChannelActionListener.onResponse(ChannelActionListener.java:43) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.action.support.ChannelActionListener.onResponse(ChannelActionListener.java:27) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.search.SearchService$3.onResponse(SearchService.java:846) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:743) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
elasticsearch_1 | "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
elasticsearch_1 | "at java.lang.Thread.run(Thread.java:832) [?:?]"] }
elasticsearch_1 | {"type": "server", "timestamp": "2021-12-13T11:08:49,321Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "elasticsearch", "node.name": "elasticsearch", "message": "path: /.suggest_analyzer/_search, params: {typed_keys=true, max_concurrent_shard_requests=5, scroll=1m, index=.suggest_analyzer, batched_reduce_size=512, ccs_minimize_roundtrips=true}", "cluster.uuid": "_3m_7YwPRGieuJDiWha8Nw", "node.id": "pnA-MHoASDahoW7Z2uZrbQ" ,
elasticsearch_1 | "stacktrace": ["org.elasticsearch.action.search.SearchPhaseExecutionException: all shards failed",
elasticsearch_1 | "at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:568) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:324) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:603) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:400) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$0(AbstractSearchAsyncAction.java:236) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.action.search.AbstractSearchAsyncAction$2.doRun(AbstractSearchAsyncAction.java:303) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:743) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.10.2.jar:7.10.2]",
elasticsearch_1 | "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
elasticsearch_1 | "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
elasticsearch_1 | "at java.lang.Thread.run(Thread.java:832) [?:?]"] }
We have now successfully migrated to FESS 13.15.2 by adopting the