When pulling metrics from dashboard, there are occurrences of failure in getting datapoints. Kairosdb log suggests that cassandra io performance may be the bottleneck, either increase number of casssandra nodes or increase the buffer may help.
11-13|21:50:00.001 [QuartzScheduler_Worker-2] INFO [WriteBuffer.java:199] - Increasing write buffer data_points size to 7469
11-13|21:50:00.107 [QuartzScheduler_Worker-2] INFO [WriteBuffer.java:199] - Increasing write buffer row_key_index size to 20189
11-13|21:50:00.107 [QuartzScheduler_Worker-2] INFO [WriteBuffer.java:199] - Increasing write buffer string_index size to 28786
11-13|21:50:25.560 [Thread-5] WARN [HConnectionManager.java:302] - Could not fullfill request on this host CassandraClient<localhost:9160-577>
11-13|21:50:25.561 [Thread-5] WARN [HConnectionManager.java:303] - Exception:
me.prettyprint.hector.api.exceptions.HTimedOutException: TimedOutException(acknowledged_by:0)
at me.prettyprint.cassandra.service.ExceptionsTranslatorImpl.translate(ExceptionsTranslatorImpl.java:42) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.connection.HConnectionManager.operateWithFailover(HConnectionManager.java:260) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.ExecutingKeyspace.doExecuteOperation(ExecutingKeyspace.java:113) [hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.MutatorImpl.execute(MutatorImpl.java:243) [hector-core-1.1-4.jar:na]
at org.kairosdb.datastore.cassandra.WriteBuffer.run(WriteBuffer.java:237) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_65]
Caused by: org.apache.cassandra.thrift.TimedOutException: null
at org.apache.cassandra.thrift.Cassandra$batch_mutate_result.read(Cassandra.java:20849) ~[cassandra-thrift-1.2.5.jar:1.2.5]
at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:78) ~[libthrift-0.7.0.jar:0.7.0]
at org.apache.cassandra.thrift.Cassandra$Client.recv_batch_mutate(Cassandra.java:964) ~[cassandra-thrift-1.2.5.jar:1.2.5]
at org.apache.cassandra.thrift.Cassandra$Client.batch_mutate(Cassandra.java:950) ~[cassandra-thrift-1.2.5.jar:1.2.5]
at me.prettyprint.cassandra.model.MutatorImpl$3.execute(MutatorImpl.java:246) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.MutatorImpl$3.execute(MutatorImpl.java:243) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.service.Operation.executeAndSetResult(Operation.java:104) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.connection.HConnectionManager.operateWithFailover(HConnectionManager.java:253) ~[hector-core-1.1-4.jar:na]
... 4 common frames omitted
11-13|21:51:40.910 [qtp122933265-76] WARN [HConnectionManager.java:302] - Could not fullfill request on this host CassandraClient<localhost:9160-588>
11-13|21:51:46.336 [qtp122933265-98] WARN [HConnectionManager.java:302] - Could not fullfill request on this host CassandraClient<localhost:9160-589>
11-13|21:51:51.577 [qtp122933265-94] WARN [HConnectionManager.java:302] - Could not fullfill request on this host CassandraClient<localhost:9160-576>
11-13|21:51:54.066 [qtp122933265-98] WARN [HConnectionManager.java:303] - Exception:
me.prettyprint.hector.api.exceptions.HTimedOutException: TimedOutException()
at me.prettyprint.cassandra.service.ExceptionsTranslatorImpl.translate(ExceptionsTranslatorImpl.java:42) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.service.KeyspaceServiceImpl$15.execute(KeyspaceServiceImpl.java:563) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.service.KeyspaceServiceImpl$15.execute(KeyspaceServiceImpl.java:549) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.service.Operation.executeAndSetResult(Operation.java:104) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.connection.HConnectionManager.operateWithFailover(HConnectionManager.java:253) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.service.KeyspaceServiceImpl.operateWithFailover(KeyspaceServiceImpl.java:132) [hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.service.KeyspaceServiceImpl.multigetSlice(KeyspaceServiceImpl.java:567) [hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.thrift.ThriftMultigetSliceQuery$1.doInKeyspace(ThriftMultigetSliceQuery.java:68) [hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.thrift.ThriftMultigetSliceQuery$1.doInKeyspace(ThriftMultigetSliceQuery.java:59) [hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.KeyspaceOperationCallback.doInKeyspaceAndMeasure(KeyspaceOperationCallback.java:20) [hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.ExecutingKeyspace.doExecute(ExecutingKeyspace.java:101) [hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.thrift.ThriftMultigetSliceQuery.execute(ThriftMultigetSliceQuery.java:58) [hector-core-1.1-4.jar:na]
at org.kairosdb.datastore.cassandra.QueryRunner.runQuery(QueryRunner.java:112) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at org.kairosdb.datastore.cassandra.CassandraDatastore.queryWithRowKeys(CassandraDatastore.java:555) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at org.kairosdb.datastore.cassandra.CassandraDatastore.queryDatabase(CassandraDatastore.java:494) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at org.kairosdb.core.datastore.KairosDatastore$DatastoreQueryImpl.execute(KairosDatastore.java:494) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at org.kairosdb.core.http.rest.MetricsResource.get(MetricsResource.java:341) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at sun.reflect.GeneratedMethodAccessor22.invoke(Unknown Source) ~[na:na]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_65]
at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_65]
at com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$ResponseOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:205) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:302) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1480) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1411) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1360) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1350) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:416) [jersey-servlet-1.15.jar:1.15]
at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:538) [jersey-servlet-1.15.jar:1.15]
at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:716) [jersey-servlet-1.15.jar:1.15]
at javax.servlet.http.HttpServlet.service(HttpServlet.java:848) [javax.servlet-3.0.0.v201112011016.jar:na]
at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:263) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:178) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.ManagedServletPipeline.service(ManagedServletPipeline.java:91) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:62) [guice-servlet-3.0.jar:na]
at org.kairosdb.core.http.MonitorFilter.doFilter(MonitorFilter.java:81) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:163) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) [guice-servlet-3.0.jar:na]
at org.kairosdb.core.http.LoggingFilter.doFilter(LoggingFilter.java:48) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:163) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:118) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:113) [guice-servlet-3.0.jar:na]
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1465) [jetty-servlet-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:499) [jetty-servlet-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1086) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:428) [jetty-servlet-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1020) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.Server.handle(Server.java:370) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:960) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1021) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:865) [jetty-http-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240) [jetty-http-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668) [jetty-io-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52) [jetty-io-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) [jetty-util-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) [jetty-util-8.1.13.v20130916.jar:8.1.13.v20130916]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_65]
Caused by: org.apache.cassandra.thrift.TimedOutException: null
at org.apache.cassandra.thrift.Cassandra$multiget_slice_result.read(Cassandra.java:10392) ~[cassandra-thrift-1.2.5.jar:1.2.5]
at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:78) ~[libthrift-0.7.0.jar:0.7.0]
at org.apache.cassandra.thrift.Cassandra$Client.recv_multiget_slice(Cassandra.java:664) ~[cassandra-thrift-1.2.5.jar:1.2.5]
at org.apache.cassandra.thrift.Cassandra$Client.multiget_slice(Cassandra.java:648) ~[cassandra-thrift-1.2.5.jar:1.2.5]
at me.prettyprint.cassandra.service.KeyspaceServiceImpl$15.execute(KeyspaceServiceImpl.java:554) ~[hector-core-1.1-4.jar:na]
... 66 common frames omitted
11-13|21:51:54.066 [qtp122933265-76] WARN [HConnectionManager.java:303] - Exception:
me.prettyprint.hector.api.exceptions.HTimedOutException: TimedOutException()
at me.prettyprint.cassandra.service.ExceptionsTranslatorImpl.translate(ExceptionsTranslatorImpl.java:42) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.service.KeyspaceServiceImpl$15.execute(KeyspaceServiceImpl.java:563) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.service.KeyspaceServiceImpl$15.execute(KeyspaceServiceImpl.java:549) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.service.Operation.executeAndSetResult(Operation.java:104) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.connection.HConnectionManager.operateWithFailover(HConnectionManager.java:253) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.service.KeyspaceServiceImpl.operateWithFailover(KeyspaceServiceImpl.java:132) [hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.service.KeyspaceServiceImpl.multigetSlice(KeyspaceServiceImpl.java:567) [hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.thrift.ThriftMultigetSliceQuery$1.doInKeyspace(ThriftMultigetSliceQuery.java:68) [hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.thrift.ThriftMultigetSliceQuery$1.doInKeyspace(ThriftMultigetSliceQuery.java:59) [hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.KeyspaceOperationCallback.doInKeyspaceAndMeasure(KeyspaceOperationCallback.java:20) [hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.ExecutingKeyspace.doExecute(ExecutingKeyspace.java:101) [hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.thrift.ThriftMultigetSliceQuery.execute(ThriftMultigetSliceQuery.java:58) [hector-core-1.1-4.jar:na]
at org.kairosdb.datastore.cassandra.QueryRunner.runQuery(QueryRunner.java:112) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at org.kairosdb.datastore.cassandra.CassandraDatastore.queryWithRowKeys(CassandraDatastore.java:555) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at org.kairosdb.datastore.cassandra.CassandraDatastore.queryDatabase(CassandraDatastore.java:494) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at org.kairosdb.core.datastore.KairosDatastore$DatastoreQueryImpl.execute(KairosDatastore.java:494) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at org.kairosdb.core.http.rest.MetricsResource.get(MetricsResource.java:341) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at sun.reflect.GeneratedMethodAccessor22.invoke(Unknown Source) ~[na:na]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_65]
at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_65]
at com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$ResponseOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:205) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:302) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1480) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1411) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1360) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1350) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:416) [jersey-servlet-1.15.jar:1.15]
at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:538) [jersey-servlet-1.15.jar:1.15]
at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:716) [jersey-servlet-1.15.jar:1.15]
at javax.servlet.http.HttpServlet.service(HttpServlet.java:848) [javax.servlet-3.0.0.v201112011016.jar:na]
at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:263) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:178) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.ManagedServletPipeline.service(ManagedServletPipeline.java:91) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:62) [guice-servlet-3.0.jar:na]
at org.kairosdb.core.http.MonitorFilter.doFilter(MonitorFilter.java:81) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:163) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) [guice-servlet-3.0.jar:na]
at org.kairosdb.core.http.LoggingFilter.doFilter(LoggingFilter.java:48) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:163) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:118) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:113) [guice-servlet-3.0.jar:na]
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1465) [jetty-servlet-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:499) [jetty-servlet-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1086) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:428) [jetty-servlet-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1020) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.Server.handle(Server.java:370) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:960) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1021) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:865) [jetty-http-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240) [jetty-http-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668) [jetty-io-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52) [jetty-io-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) [jetty-util-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) [jetty-util-8.1.13.v20130916.jar:8.1.13.v20130916]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_65]
Caused by: org.apache.cassandra.thrift.TimedOutException: null
at org.apache.cassandra.thrift.Cassandra$multiget_slice_result.read(Cassandra.java:10392) ~[cassandra-thrift-1.2.5.jar:1.2.5]
at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:78) ~[libthrift-0.7.0.jar:0.7.0]
at org.apache.cassandra.thrift.Cassandra$Client.recv_multiget_slice(Cassandra.java:664) ~[cassandra-thrift-1.2.5.jar:1.2.5]
at org.apache.cassandra.thrift.Cassandra$Client.multiget_slice(Cassandra.java:648) ~[cassandra-thrift-1.2.5.jar:1.2.5]
at me.prettyprint.cassandra.service.KeyspaceServiceImpl$15.execute(KeyspaceServiceImpl.java:554) ~[hector-core-1.1-4.jar:na]
... 66 common frames omitted
11-13|21:51:54.066 [qtp122933265-94] WARN [HConnectionManager.java:303] - Exception:
me.prettyprint.hector.api.exceptions.HTimedOutException: TimedOutException()
at me.prettyprint.cassandra.service.ExceptionsTranslatorImpl.translate(ExceptionsTranslatorImpl.java:42) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.service.KeyspaceServiceImpl$15.execute(KeyspaceServiceImpl.java:563) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.service.KeyspaceServiceImpl$15.execute(KeyspaceServiceImpl.java:549) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.service.Operation.executeAndSetResult(Operation.java:104) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.connection.HConnectionManager.operateWithFailover(HConnectionManager.java:253) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.service.KeyspaceServiceImpl.operateWithFailover(KeyspaceServiceImpl.java:132) [hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.service.KeyspaceServiceImpl.multigetSlice(KeyspaceServiceImpl.java:567) [hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.thrift.ThriftMultigetSliceQuery$1.doInKeyspace(ThriftMultigetSliceQuery.java:68) [hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.thrift.ThriftMultigetSliceQuery$1.doInKeyspace(ThriftMultigetSliceQuery.java:59) [hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.KeyspaceOperationCallback.doInKeyspaceAndMeasure(KeyspaceOperationCallback.java:20) [hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.ExecutingKeyspace.doExecute(ExecutingKeyspace.java:101) [hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.thrift.ThriftMultigetSliceQuery.execute(ThriftMultigetSliceQuery.java:58) [hector-core-1.1-4.jar:na]
at org.kairosdb.datastore.cassandra.QueryRunner.runQuery(QueryRunner.java:112) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at org.kairosdb.datastore.cassandra.CassandraDatastore.queryWithRowKeys(CassandraDatastore.java:555) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at org.kairosdb.datastore.cassandra.CassandraDatastore.queryDatabase(CassandraDatastore.java:494) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at org.kairosdb.core.datastore.KairosDatastore$DatastoreQueryImpl.execute(KairosDatastore.java:494) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at org.kairosdb.core.http.rest.MetricsResource.get(MetricsResource.java:341) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at sun.reflect.GeneratedMethodAccessor22.invoke(Unknown Source) ~[na:na]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_65]
at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_65]
at com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$ResponseOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:205) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:302) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1480) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1411) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1360) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1350) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:416) [jersey-servlet-1.15.jar:1.15]
at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:538) [jersey-servlet-1.15.jar:1.15]
at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:716) [jersey-servlet-1.15.jar:1.15]
at javax.servlet.http.HttpServlet.service(HttpServlet.java:848) [javax.servlet-3.0.0.v201112011016.jar:na]
at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:263) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:178) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.ManagedServletPipeline.service(ManagedServletPipeline.java:91) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:62) [guice-servlet-3.0.jar:na]
at org.kairosdb.core.http.MonitorFilter.doFilter(MonitorFilter.java:81) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:163) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) [guice-servlet-3.0.jar:na]
at org.kairosdb.core.http.LoggingFilter.doFilter(LoggingFilter.java:48) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:163) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:118) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:113) [guice-servlet-3.0.jar:na]
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1465) [jetty-servlet-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:499) [jetty-servlet-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1086) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:428) [jetty-servlet-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1020) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.Server.handle(Server.java:370) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:960) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1021) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:865) [jetty-http-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240) [jetty-http-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668) [jetty-io-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52) [jetty-io-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) [jetty-util-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) [jetty-util-8.1.13.v20130916.jar:8.1.13.v20130916]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_65]
Caused by: org.apache.cassandra.thrift.TimedOutException: null
at org.apache.cassandra.thrift.Cassandra$multiget_slice_result.read(Cassandra.java:10392) ~[cassandra-thrift-1.2.5.jar:1.2.5]
at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:78) ~[libthrift-0.7.0.jar:0.7.0]
at org.apache.cassandra.thrift.Cassandra$Client.recv_multiget_slice(Cassandra.java:664) ~[cassandra-thrift-1.2.5.jar:1.2.5]
at org.apache.cassandra.thrift.Cassandra$Client.multiget_slice(Cassandra.java:648) ~[cassandra-thrift-1.2.5.jar:1.2.5]
at me.prettyprint.cassandra.service.KeyspaceServiceImpl$15.execute(KeyspaceServiceImpl.java:554) ~[hector-core-1.1-4.jar:na]
... 66 common frames omitted
11-13|21:52:01.648 [qtp122933265-98] ERROR [MetricsResource.java:393] - Query failed.
org.kairosdb.core.exception.DatastoreException: me.prettyprint.hector.api.exceptions.HTimedOutException: TimedOutException()
at org.kairosdb.core.datastore.KairosDatastore$DatastoreQueryImpl.execute(KairosDatastore.java:500) ~[kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at org.kairosdb.core.http.rest.MetricsResource.get(MetricsResource.java:341) ~[kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at sun.reflect.GeneratedMethodAccessor22.invoke(Unknown Source) ~[na:na]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_65]
at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_65]
at com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$ResponseOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:205) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:302) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1480) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1411) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1360) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1350) [jersey-server-1.15.jar:1.15]
at com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:416) [jersey-servlet-1.15.jar:1.15]
at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:538) [jersey-servlet-1.15.jar:1.15]
at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:716) [jersey-servlet-1.15.jar:1.15]
at javax.servlet.http.HttpServlet.service(HttpServlet.java:848) [javax.servlet-3.0.0.v201112011016.jar:na]
at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:263) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:178) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.ManagedServletPipeline.service(ManagedServletPipeline.java:91) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:62) [guice-servlet-3.0.jar:na]
at org.kairosdb.core.http.MonitorFilter.doFilter(MonitorFilter.java:81) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:163) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) [guice-servlet-3.0.jar:na]
at org.kairosdb.core.http.LoggingFilter.doFilter(LoggingFilter.java:48) [kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:163) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:118) [guice-servlet-3.0.jar:na]
at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:113) [guice-servlet-3.0.jar:na]
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1465) [jetty-servlet-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:499) [jetty-servlet-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1086) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:428) [jetty-servlet-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1020) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.Server.handle(Server.java:370) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:960) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1021) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:865) [jetty-http-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240) [jetty-http-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82) [jetty-server-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668) [jetty-io-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52) [jetty-io-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) [jetty-util-8.1.13.v20130916.jar:8.1.13.v20130916]
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) [jetty-util-8.1.13.v20130916.jar:8.1.13.v20130916]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_65]
Caused by: me.prettyprint.hector.api.exceptions.HTimedOutException: TimedOutException()
at me.prettyprint.cassandra.service.ExceptionsTranslatorImpl.translate(ExceptionsTranslatorImpl.java:42) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.service.KeyspaceServiceImpl$15.execute(KeyspaceServiceImpl.java:563) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.service.KeyspaceServiceImpl$15.execute(KeyspaceServiceImpl.java:549) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.service.Operation.executeAndSetResult(Operation.java:104) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.connection.HConnectionManager.operateWithFailover(HConnectionManager.java:253) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.service.KeyspaceServiceImpl.operateWithFailover(KeyspaceServiceImpl.java:132) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.service.KeyspaceServiceImpl.multigetSlice(KeyspaceServiceImpl.java:567) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.thrift.ThriftMultigetSliceQuery$1.doInKeyspace(ThriftMultigetSliceQuery.java:68) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.thrift.ThriftMultigetSliceQuery$1.doInKeyspace(ThriftMultigetSliceQuery.java:59) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.KeyspaceOperationCallback.doInKeyspaceAndMeasure(KeyspaceOperationCallback.java:20) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.ExecutingKeyspace.doExecute(ExecutingKeyspace.java:101) ~[hector-core-1.1-4.jar:na]
at me.prettyprint.cassandra.model.thrift.ThriftMultigetSliceQuery.execute(ThriftMultigetSliceQuery.java:58) ~[hector-core-1.1-4.jar:na]
at org.kairosdb.datastore.cassandra.QueryRunner.runQuery(QueryRunner.java:112) ~[kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at org.kairosdb.datastore.cassandra.CassandraDatastore.queryWithRowKeys(CassandraDatastore.java:555) ~[kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at org.kairosdb.datastore.cassandra.CassandraDatastore.queryDatabase(CassandraDatastore.java:494) ~[kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
at org.kairosdb.core.datastore.KairosDatastore$DatastoreQueryImpl.execute(KairosDatastore.java:494) ~[kairosdb-0.9.4-6.jar:0.9.4-6.20140730155353]
... 52 common frames omitted
Caused by: org.apache. cassandra. thrift. TimedOutExcepti on: null cassandra. thrift. Cassandra$ multiget_ slice_result. read(Cassandra. java:10392) ~[cassandra- thrift- 1.2.5.jar: 1.2.5] thrift. TServiceClient. receiveBase( TServiceClient. java:78) ~[libthrift- 0.7.0.jar: 0.7.0] cassandra. thrift. Cassandra$ Client. recv_multiget_ slice(Cassandra .java:664) ~[cassandra- thrift- 1.2.5.jar: 1.2.5] cassandra. thrift. Cassandra$ Client. multiget_ slice(Cassandra .java:648) ~[cassandra- thrift- 1.2.5.jar: 1.2.5] cassandra. service. KeyspaceService Impl$15. execute( KeyspaceService Impl.java: 554) ~[hector- core-1. 1-4.jar: na] e.java: 393] - Query failed. core.exception. DatastoreExcept ion: me.prettyprint. hector. api.exceptions. HTimedOutExcept ion: TimedOutException() core.datastore. KairosDatastore $DatastoreQuery Impl.execute( KairosDatastore .java:500) ~[kairosdb- 0.9.4-6. jar:0.9. 4-6.20140730155 353] core.http. rest.MetricsRes ource.get( MetricsResource .java:341) ~[kairosdb- 0.9.4-6. jar:0.9. 4-6.20140730155 353] GeneratedMethod Accessor22. invoke( Unknown Source) ~[na:na] DelegatingMetho dAccessorImpl. invoke( DelegatingMetho dAccessorImpl. java:43) ~[na:1.7.0_65] reflect. Method. invoke( Method. java:606) ~[na:1.7.0_65] jersey. spi.container. JavaMethodInvok erFactory$ 1.invoke( JavaMethodInvok erFactory. java:60) [jersey- server- 1.15.jar: 1.15] jersey. server. impl.model. method. dispatch. AbstractResourc eMethodDispatch Provider$ ResponseOutInvo ker._dispatch( AbstractResourc eMethodDispatch Provider. java:205) [jersey- server- 1.15.jar: 1.15] jersey. server. impl.model. method. dispatch. ResourceJavaMet hodDispatcher. dispatch( ResourceJavaMet hodDispatcher. java:75) [jersey- server- 1.15.jar: 1.15] jersey. server. impl.uri. rules.HttpMetho dRule.accept( HttpMethodRule. java:302) [jersey- server- 1.15.jar: 1.15] jersey. server. impl.uri. rules.RightHand PathRule. accept( RightHandPathRu le.java: 147) [jersey- server- 1.15.jar: 1.15] jersey. server. impl.uri. rules.ResourceC lassRule. accept( ResourceClassRu le.java: 108) [jersey- server- 1.15.jar: 1.15] jersey. server. impl.uri. rules.RightHand PathRule. accept( RightHandPathRu le.java: 147) [jersey- server- 1.15.jar: 1.15] jersey. server. impl.uri. rules.RootResou rceClassesRule. accept( RootResourceCla ssesRule. java:84) [jersey- server- 1.15.jar: 1.15] jersey. server. impl.applicatio n.WebApplicatio nImpl._ handleRequest( WebApplicationI mpl.java: 1480) [jersey- server- 1.15.jar: 1.15] jersey. server. impl.applicatio n.WebApplicatio nImpl._ handleRequest( WebApplicationI mpl.java: 1411) [jersey- server- 1.15.jar: 1.15] jersey. server. impl.applicatio n.WebApplicatio nImpl.handleReq uest(WebApplica tionImpl. java:1360) [jersey- server- 1.15.jar: 1.15] jersey. server. impl.applicatio n.WebApplicatio nImpl.handleReq uest(WebApplica tionImpl. java:1350) [jersey- server- 1.15.jar: 1.15] jersey. spi.container. servlet. WebComponent. service( WebComponent. java:416) [jersey- servlet- 1.15.jar: 1.15].. .
at org.apache.
at org.apache.
at org.apache.
at org.apache.
at me.prettyprint.
... 66 common frames omitted
11-13|21:52:01.649 [qtp122933265-76] ERROR [MetricsResourc
org.kairosdb.
at org.kairosdb.
at org.kairosdb.
at sun.reflect.
at sun.reflect.
at java.lang.
at com.sun.
at com.sun.
at com.sun.
at com.sun.
at com.sun.
at com.sun.
at com.sun.
at com.sun.
at com.sun.
at com.sun.
at com.sun.
at com.sun.
at com.sun.