Ignition 8.1.30
PostgreSQL Database
Available storage: Over 150 GB (dedicated drive)
DB Connections: 10/10
Over the last couple weeks, the following error has flooded our gateway logs more and more frequently. Hundreds of log errors can be generated in less than a minute. What could be a potential cause or solution? Could viewing elaborate power charts be responsible? Anything else?
So far the issue resolves itself, however each occurrence is lasting longer and longer. No known side effects have been noticed through the perspective front end, however the errors are trending in the wrong direction.
Any help appreciated!
query Error querying historical annotations for provider '{histprov/LocalDatabase}'.
com.inductiveautomation.ignition.gateway.datasource.FaultedDatabaseConnectionException: The database connection 'LocalDatabase' is FAULTED. See Gateway Status for details.
at com.inductiveautomation.ignition.gateway.datasource.DatasourceManagerImpl.getConnectionImpl(DatasourceManagerImpl.java:211)
at com.inductiveautomation.ignition.gateway.datasource.DatasourceImpl.getConnection(DatasourceImpl.java:293)
at com.inductiveautomation.ignition.gateway.datasource.DatasourceManagerImpl.getConnection(DatasourceManagerImpl.java:152)
at com.inductiveautomation.gateway.tags.history.processing.GatewayContextConnectionProvider.getConnection(GatewayContextConnectionProvider.java:31)
at com.inductiveautomation.gateway.tags.history.processing.GatewayContextConnectionProvider.getConnection(GatewayContextConnectionProvider.java:10)
at com.inductiveautomation.gateway.tags.history.query.DatasourceHistoryInterface.getConnection(DatasourceHistoryInterface.java:77)
at com.inductiveautomation.gateway.tags.history.query.DatasourceHistoryInterface.maybeLoadIds(DatasourceHistoryInterface.java:96)
at com.inductiveautomation.gateway.tags.history.query.DatasourceHistoryInterface.loadTagInfo(DatasourceHistoryInterface.java:229)
at com.inductiveautomation.gateway.tags.history.query.DatasourceAnnotationQuery.execute(DatasourceAnnotationQuery.java:99)
at com.inductiveautomation.gateway.tags.history.module.TagHistoryProvider.queryAnnotations(TagHistoryProvider.java:210)
at com.inductiveautomation.ignition.gateway.sqltags.history.TagHistoryManagerImpl.queryAnnotations(TagHistoryManagerImpl.java:699)
at com.inductiveautomation.ignition.gateway.tags.model.ProjectDefaultTagManagerFacade.queryAnnotations(ProjectDefaultTagManagerFacade.java:453)
at com.inductiveautomation.perspective.gateway.components.PowerChartModelDelegate$AnnotationFetch.fetch(PowerChartModelDelegate.java:278)
at com.inductiveautomation.perspective.gateway.comm.FetchableCacheImpl.lambda$fetch$0(FetchableCacheImpl.java:47)
at com.inductiveautomation.perspective.gateway.threading.BlockingWork.invokeBlockingTask(BlockingWork.java:34)
at com.inductiveautomation.perspective.gateway.comm.FetchableCacheImpl.fetch(FetchableCacheImpl.java:47)
at com.inductiveautomation.perspective.gateway.comm.Routes.handleFetch(Routes.java:2371)
at com.inductiveautomation.ignition.gateway.dataroutes.Route.service(Route.java:254)
at com.inductiveautomation.ignition.gateway.dataroutes.RouteGroupImpl.service(RouteGroupImpl.java:61)
at com.inductiveautomation.ignition.gateway.dataroutes.RouteGroupCollectionServlet.serviceInternal(RouteGroupCollectionServlet.java:59)
at com.inductiveautomation.ignition.gateway.dataroutes.AbstractRouteGroupServlet.service(AbstractRouteGroupServlet.java:38)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
at org.eclipse.jetty.servlet.ServletHolder$NotAsync.service(ServletHolder.java:1410)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1384)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1306)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
at com.inductiveautomation.catapult.handlers.RemoteHostNameLookupHandler.handle(RemoteHostNameLookupHandler.java:121)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:301)
at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:51)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:141)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
at org.eclipse.jetty.server.Server.handle(Server.java:563)
at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.runTask(AdaptiveExecutionStrategy.java:416)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.consumeTask(AdaptiveExecutionStrategy.java:385)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:272)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.lambda$new$0(AdaptiveExecutionStrategy.java:140)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:411)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:969)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1194)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1149)
at java.base/java.lang.Thread.run(Unknown Source)
Beginning of log message can also read:
query Error querying historical annotations for provider '{histprov/LocalDatabase}'. com.inductiveautomation.ignition.gateway.datasource.DisabledDatabaseConnectionException: The database connection 'LocalDatabase' is disabled.
at com.inductiveautomation.ignition.gateway.datasource.DatasourceManagerImpl.getConnectionImpl(DatasourceManagerImpl.java:162)