Hi,
I do have a application hosted in AWS Beanstalk environment which has been built using CUBA 7.2. It used to work smoothly. But today, when user trying to add a record in a screen which contains Grids and auto save using timer, Users are getting server connection lost message when a program create multiple records and adds them to the container.
While this happens the server is not accepting any new connection and through 404, the CPU usage goes to 93-95%.
And once the user closed the tab in chrome after few minutes the server is responding normaly.
I did check the log and found the below error repeating.
16:41:52.496 WARN c.h.cuba.core.jmx.StatisticsCounter - Can't get MBean attribute maxTotal from Catalina:class=javax.sql.DataSource,context=/,host=localhost,name="jdbc/CubaDS",type=DataSource pool!
javax.management.AttributeNotFoundException: Cannot find attribute maxTotal for org.apache.tomcat.jdbc.pool.DataSource@6ccb0ea7{ConnectionPool[defaultAutoCommit=null; defaultReadOnly=null; defaultTransactionIsolation=-1; defaultCatalog=null; driverClassName=org.postgresql.Driver; maxActive=100; maxIdle=10; minIdle=10; initialSize=10; maxWait=30000; testOnBorrow=false; testOnReturn=false; timeBetweenEvictionRunsMillis=5000; numTestsPerEvictionRun=0; minEvictableIdleTimeMillis=60000; testWhileIdle=false; testOnConnect=false; password=********; url=jdbc:postgresql://score.cqrrwvyypqjs.ap-south-1.rds.amazonaws.com/scoringsystem; username=scoringsystem; validationQuery=null; validationQueryTimeout=-1; validatorClassName=null; validationInterval=3000; accessToUnderlyingConnectionAllowed=true; removeAbandoned=false; removeAbandonedTimeout=60; logAbandoned=false; connectionProperties=null; initSQL=null; jdbcInterceptors=null; jmxEnabled=true; fairQueue=true; useEquals=true; abandonWhenPercentageFull=0; maxAge=0; useLock=false; dataSource=null; dataSourceJNDI=null; suspectTimeout=0; alternateUsernameAllowed=false; commitOnReturn=false; rollbackOnReturn=false; useDisposableConnectionFacade=true; logValidationErrors=false; propagateInterruptState=false; ignoreExceptionOnPreLoad=false; useStatementFacade=true; }
at org.apache.tomcat.util.modeler.ManagedBean.getGetter(ManagedBean.java:435) ~[tomcat-coyote.jar:8.5.57]
at org.apache.tomcat.util.modeler.BaseModelMBean.getAttribute(BaseModelMBean.java:166) ~[tomcat-coyote.jar:8.5.57]
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getAttribute(DefaultMBeanServerInterceptor.java:647) [na:1.8.0_265]
at com.sun.jmx.mbeanserver.JmxMBeanServer.getAttribute(JmxMBeanServer.java:678) [na:1.8.0_265]
at com.haulmont.cuba.core.jmx.StatisticsCounter.getDbConnectionPoolMBeanAttr(StatisticsCounter.java:149) [cuba-core-7.2.7.jar:7.2.7]
at com.haulmont.cuba.core.jmx.StatisticsCounter.getDbConnectionPoolMaxTotal(StatisticsCounter.java:124) [cuba-core-7.2.7.jar:7.2.7]
at sun.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[na:na]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_265]
at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_265]
at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:344) [spring-aop-5.2.3.RELEASE.jar:5.2.3.RELEASE]
at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:198) [spring-aop-5.2.3.RELEASE.jar:5.2.3.RELEASE]
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163) [spring-aop-5.2.3.RELEASE.jar:5.2.3.RELEASE]
at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:88) [spring-aop-5.2.3.RELEASE.jar:5.2.3.RELEASE]
at com.haulmont.cuba.core.sys.MBeanInterceptor.aroundInvoke(MBeanInterceptor.java:39) [cuba-core-7.2.7.jar:7.2.7]
at sun.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[na:na]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_265]
at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_265]
at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:644) [spring-aop-5.2.3.RELEASE.jar:5.2.3.RELEASE]
at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:633) [spring-aop-5.2.3.RELEASE.jar:5.2.3.RELEASE]
at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:70) [spring-aop-5.2.3.RELEASE.jar:5.2.3.RELEASE]
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) [spring-aop-5.2.3.RELEASE.jar:5.2.3.RELEASE]
at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:95) [spring-aop-5.2.3.RELEASE.jar:5.2.3.RELEASE]
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) [spring-aop-5.2.3.RELEASE.jar:5.2.3.RELEASE]
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:212) [spring-aop-5.2.3.RELEASE.jar:5.2.3.RELEASE]
at com.sun.proxy.$Proxy164.getDbConnectionPoolMaxTotal(Unknown Source) [na:na]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.8.0_265]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:1.8.0_265]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_265]
at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_265]
at sun.reflect.misc.Trampoline.invoke(MethodUtil.java:72) [na:1.8.0_265]
at sun.reflect.GeneratedMethodAccessor90.invoke(Unknown Source) ~[na:na]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_265]
at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_265]
at sun.reflect.misc.MethodUtil.invoke(MethodUtil.java:276) [na:1.8.0_265]
at javax.management.modelmbean.RequiredModelMBean$4.run(RequiredModelMBean.java:1252) [na:1.8.0_265]
at java.security.AccessController.doPrivileged(Native Method) [na:1.8.0_265]
at java.security.ProtectionDomain$JavaSecurityAccessImpl.doIntersectionPrivilege(ProtectionDomain.java:74) [na:1.8.0_265]
at javax.management.modelmbean.RequiredModelMBean.invokeMethod(RequiredModelMBean.java:1246) [na:1.8.0_265]
at javax.management.modelmbean.RequiredModelMBean.invoke(RequiredModelMBean.java:1085) [na:1.8.0_265]
at org.springframework.jmx.export.SpringModelMBean.invoke(SpringModelMBean.java:90) [spring-context-5.2.3.RELEASE.jar:5.2.3.RELEASE]
at javax.management.modelmbean.RequiredModelMBean.getAttribute(RequiredModelMBean.java:1562) [na:1.8.0_265]
at org.springframework.jmx.export.SpringModelMBean.getAttribute(SpringModelMBean.java:109) [spring-context-5.2.3.RELEASE.jar:5.2.3.RELEASE]
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getAttribute(DefaultMBeanServerInterceptor.java:647) [na:1.8.0_265]
at com.sun.jmx.mbeanserver.JmxMBeanServer.getAttribute(JmxMBeanServer.java:678) [na:1.8.0_265]
at com.haulmont.cuba.web.jmx.JmxControlBean.lambda$loadAttributeValue$5(JmxControlBean.java:296) [cuba-web-7.2.7.jar:7.2.7]
at com.haulmont.cuba.web.jmx.JmxConnectionHelper.withConnection(JmxConnectionHelper.java:105) ~[cuba-web-7.2.7.jar:7.2.7]
at com.haulmont.cuba.web.jmx.JmxControlBean.loadAttributeValue(JmxControlBean.java:290) [cuba-web-7.2.7.jar:7.2.7]
at com.haulmont.cuba.web.app.ui.statistics.StatisticsDatasource.findAttribute(StatisticsDatasource.java:128) ~[cuba-web-7.2.7.jar:7.2.7]
at com.haulmont.cuba.web.app.ui.statistics.StatisticsDatasource.loadCurrentValues(StatisticsDatasource.java:252) ~[cuba-web-7.2.7.jar:7.2.7]
at com.haulmont.cuba.web.app.ui.statistics.StatisticsDatasource.loadData(StatisticsDatasource.java:105) ~[cuba-web-7.2.7.jar:7.2.7]
at com.haulmont.cuba.gui.data.impl.CollectionDatasourceImpl.refresh(CollectionDatasourceImpl.java:165) ~[cuba-gui-7.2.7.jar:7.2.7]
at com.haulmont.cuba.gui.data.impl.CollectionDatasourceImpl.refresh(CollectionDatasourceImpl.java:125) ~[cuba-gui-7.2.7.jar:7.2.7]
at com.haulmont.cuba.web.app.ui.statistics.StatisticsWindow.onRefresh(StatisticsWindow.java:125) ~[cuba-web-7.2.7.jar:7.2.7]
at sun.reflect.GeneratedMethodAccessor384.invoke(Unknown Source) ~[na:na]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_265]
at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_265]
at com.haulmont.cuba.web.gui.facets.TimerFacetProvider$DeclarativeTimerActionHandler.accept(TimerFacetProvider.java:138) ~[cuba-web-7.2.7.jar:7.2.7]
at com.haulmont.cuba.web.gui.facets.TimerFacetProvider$DeclarativeTimerActionHandler.accept(TimerFacetProvider.java:125) ~[cuba-web-7.2.7.jar:7.2.7]
at com.haulmont.cuba.web.gui.components.WebTimer$CubaTimerActionListenerWrapper.accept(WebTimer.java:185) ~[cuba-web-7.2.7.jar:7.2.7]
at com.haulmont.cuba.web.gui.components.WebTimer$CubaTimerActionListenerWrapper.accept(WebTimer.java:174) ~[cuba-web-7.2.7.jar:7.2.7]
at com.haulmont.cuba.web.widgets.CubaTimer.onTimer(CubaTimer.java:105) ~[cuba-web-widgets-7.2.7.jar:na]
at sun.reflect.GeneratedMethodAccessor249.invoke(Unknown Source) ~[na:na]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_265]
at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_265]
at com.vaadin.server.ServerRpcManager.applyInvocation(ServerRpcManager.java:153) ~[vaadin-server-8.9.2-14-cuba.jar:8.9.2-14-cuba]
at com.vaadin.server.ServerRpcManager.applyInvocation(ServerRpcManager.java:115) ~[vaadin-server-8.9.2-14-cuba.jar:8.9.2-14-cuba]
at com.vaadin.server.communication.ServerRpcHandler.handleInvocation(ServerRpcHandler.java:431) ~[vaadin-server-8.9.2-14-cuba.jar:8.9.2-14-cuba]
at com.vaadin.server.communication.ServerRpcHandler.handleInvocations(ServerRpcHandler.java:396) ~[vaadin-server-8.9.2-14-cuba.jar:8.9.2-14-cuba]
at com.vaadin.server.communication.ServerRpcHandler.handleRpc(ServerRpcHandler.java:260) ~[vaadin-server-8.9.2-14-cuba.jar:8.9.2-14-cuba]
at com.vaadin.server.communication.UidlRequestHandler.synchronizedHandleRequest(UidlRequestHandler.java:82) ~[vaadin-server-8.9.2-14-cuba.jar:8.9.2-14-cuba]
at com.vaadin.server.SynchronizedRequestHandler.handleRequest(SynchronizedRequestHandler.java:40) ~[vaadin-server-8.9.2-14-cuba.jar:8.9.2-14-cuba]
at com.vaadin.server.VaadinService.handleRequest(VaadinService.java:1578) ~[vaadin-server-8.9.2-14-cuba.jar:8.9.2-14-cuba]
at com.vaadin.server.VaadinServlet.service(VaadinServlet.java:425) ~[vaadin-server-8.9.2-14-cuba.jar:8.9.2-14-cuba]
at com.haulmont.cuba.web.sys.CubaApplicationServlet.serviceAppRequest(CubaApplicationServlet.java:329) ~[cuba-web-7.2.7.jar:7.2.7]
at com.haulmont.cuba.web.sys.CubaApplicationServlet.service(CubaApplicationServlet.java:215) ~[cuba-web-7.2.7.jar:7.2.7]
at javax.servlet.http.HttpServlet.service(HttpServlet.java:733) ~[tomcat8-servlet-3.1-api.jar:na]
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231) ~[catalina.jar:8.5.57]
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) ~[catalina.jar:8.5.57]
at org.springframework.web.filter.CompositeFilter$VirtualFilterChain.doFilter(CompositeFilter.java:108) ~[spring-web-5.2.3.RELEASE.jar:5.2.3.RELEASE]
at org.springframework.web.filter.CompositeFilter.doFilter(CompositeFilter.java:74) ~[spring-web-5.2.3.RELEASE.jar:5.2.3.RELEASE]
at com.haulmont.cuba.web.sys.CubaHttpFilter.doFilter(CubaHttpFilter.java:93) ~[cuba-web-7.2.7.jar:7.2.7]
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) ~[catalina.jar:8.5.57]
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) ~[catalina.jar:8.5.57]
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52) ~[tomcat-websocket.jar:8.5.57]
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) ~[catalina.jar:8.5.57]
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) ~[catalina.jar:8.5.57]
at com.haulmont.cuba.web.sys.singleapp.SingleAppWebContextLoader$SetClassLoaderFilter.doFilter(SingleAppWebContextLoader.java:257) ~[cuba-web-7.2.7.jar:7.2.7]
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) ~[catalina.jar:8.5.57]
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) ~[catalina.jar:8.5.57]
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:199) ~[catalina.jar:8.5.57]
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96) ~[catalina.jar:8.5.57]
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:543) ~[catalina.jar:8.5.57]
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:139) ~[catalina.jar:8.5.57]
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:81) ~[catalina.jar:8.5.57]
at org.apache.catalina.valves.RemoteIpValve.invoke(RemoteIpValve.java:747) ~[catalina.jar:8.5.57]
at org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:690) ~[catalina.jar:8.5.57]
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:87) ~[catalina.jar:8.5.57]
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:343) ~[catalina.jar:8.5.57]
at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:615) ~[tomcat-coyote.jar:8.5.57]
at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65) ~[tomcat-coyote.jar:8.5.57]
at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:818) ~[tomcat-coyote.jar:8.5.57]
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1626) ~[tomcat-coyote.jar:8.5.57]
at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49) ~[tomcat-coyote.jar:8.5.57]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_265]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_265]
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) ~[tomcat-util.jar:8.5.57]
at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_265]
I’m attaching the catalina.log file here for the reference.
catalina.txt (7.2 MB)
Update:
Sometimes below GC overhead error is also coming, something wrong for sure.
Kindly help me fix this error.
Thanks,
Hari