(DEPRECATED) Apache Flink User Mailing List archive.

The heartbeat of TaskManager with id ... timed out.

Classic

List

Threaded

1 message

Hao Sun

The heartbeat of TaskManager with id ... timed out.

I am running Flink 1.7 on K8S. I am not sure how to debug this issue. I turned on debug on JM/TM.

I am not sure this part is related or not. How could an Actor suddenly disappear?

=====

2018-11-09 04:47:19,480 DEBUG org.apache.flink.runtime.rest.handler.legacy.metrics.MetricFetcher - Query metrics for akka://flink-metrics/user/MetricQueryService.

2018-11-09 04:47:19,577 DEBUG org.apache.flink.runtime.rest.handler.legacy.metrics.MetricFetcher - Retrieve metric query service gateway for akka.tcp://flink-metrics@fps-flink-taskmanager-6f8f687fc8-b5qmm:34429/user/MetricQueryService_0cfaa7b8f193a8002f121282298c58ac

2018-11-09 04:47:19,577 DEBUG org.apache.flink.runtime.rest.handler.legacy.metrics.MetricFetcher - Retrieve metric query service gateway for akka.tcp://flink-metrics@fps-flink-taskmanager-6f8f687fc8-ckwzp:43062/user/MetricQueryService_1930889e5b6c51cbe57428f9a664e4dc

2018-11-09 04:47:19,578 DEBUG org.apache.flink.runtime.rest.handler.legacy.metrics.MetricFetcher - Retrieve metric query service gateway for akka.tcp://flink-metrics@fps-flink-taskmanager-6f8f687fc8-bzq2w:39393/user/MetricQueryService_fcaf62301df0a6aeb29a65470cfe1e7a

2018-11-09 04:47:19,585 TRACE org.apache.flink.runtime.rest.FileUploadHandler - Received request. URL:/jobs/00000000000000000000000000000000 Method:GET

2018-11-09 04:47:19,613 DEBUG org.apache.flink.runtime.rest.handler.legacy.metrics.MetricFetcher - Could not retrieve QueryServiceGateway.

java.util.concurrent.CompletionException: akka.actor.ActorNotFound: Actor not found for: ActorSelection[Anchor(akka.tcp://flink-metrics@fps-flink-taskmanager-6f8f687fc8-ckwzp:43062/), Path(/user/MetricQueryService_1930889e5b6c51cbe57428f9a664e4dc)]

at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:292)

at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:308)

at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:593)

at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577)

at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474)

at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1977)

at org.apache.flink.runtime.concurrent.FutureUtils$1.onComplete(FutureUtils.java:772)

at akka.dispatch.OnComplete.internal(Future.scala:258)

at akka.dispatch.OnComplete.internal(Future.scala:256)

at akka.dispatch.japi$CallbackBridge.apply(Future.scala:186)

at akka.dispatch.japi$CallbackBridge.apply(Future.scala:183)

at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:60)

at org.apache.flink.runtime.concurrent.Executors$DirectExecutionContext.execute(Executors.java:83)

at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:68)

at scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1(Promise.scala:284)

at scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1$adapted(Promise.scala:284)

at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:284)

at scala.concurrent.Promise.complete(Promise.scala:49)

at scala.concurrent.Promise.complete$(Promise.scala:48)

at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:183)

at scala.concurrent.Promise.failure(Promise.scala:100)

at scala.concurrent.Promise.failure$(Promise.scala:100)

at scala.concurrent.impl.Promise$DefaultPromise.failure(Promise.scala:183)

at akka.actor.ActorSelection.$anonfun$resolveOne$1(ActorSelection.scala:68)

at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:60)

at akka.dispatch.BatchingExecutor$AbstractBatch.processBatch(BatchingExecutor.scala:55)

at akka.dispatch.BatchingExecutor$Batch.run(BatchingExecutor.scala:73)

at akka.dispatch.ExecutionContexts$sameThreadExecutionContext$.unbatchedExecute(Future.scala:76)

at akka.dispatch.BatchingExecutor.execute(BatchingExecutor.scala:120)

at akka.dispatch.BatchingExecutor.execute$(BatchingExecutor.scala:114)

at akka.dispatch.ExecutionContexts$sameThreadExecutionContext$.execute(Future.scala:75)

at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:68)

at scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1(Promise.scala:284)

at scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1$adapted(Promise.scala:284)

at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:284)

at akka.pattern.PromiseActorRef.$bang(AskSupport.scala:538)

at akka.actor.EmptyLocalActorRef.specialHandle(ActorRef.scala:558)

at akka.actor.DeadLetterActorRef.specialHandle(ActorRef.scala:595)

at akka.actor.DeadLetterActorRef.$bang(ActorRef.scala:584)

at akka.remote.RemoteActorRefProvider$RemoteDeadLetterActorRef.$bang(RemoteActorRefProvider.scala:98)

at akka.remote.EndpointWriter.postStop(Endpoint.scala:593)

at akka.actor.Actor.aroundPostStop(Actor.scala:515)

at akka.actor.Actor.aroundPostStop$(Actor.scala:515)

at akka.remote.EndpointActor.aroundPostStop(Endpoint.scala:446)

at akka.actor.dungeon.FaultHandling.finishTerminate(FaultHandling.scala:210)

at akka.actor.dungeon.FaultHandling.terminate(FaultHandling.scala:172)

at akka.actor.dungeon.FaultHandling.terminate$(FaultHandling.scala:142)

at akka.actor.ActorCell.terminate(ActorCell.scala:374)

at akka.actor.ActorCell.invokeAll$1(ActorCell.scala:467)

at akka.actor.ActorCell.systemInvoke(ActorCell.scala:483)

at akka.dispatch.Mailbox.processAllSystemMessages(Mailbox.scala:282)

at akka.dispatch.Mailbox.run(Mailbox.scala:223)

at akka.dispatch.Mailbox.exec(Mailbox.scala:234)

at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)

at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)

at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)

at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:157)

Caused by: akka.actor.ActorNotFound: Actor not found for: ActorSelection[Anchor(akka.tcp://flink-metrics@fps-flink-taskmanager-6f8f687fc8-ckwzp:43062/), Path(/user/MetricQueryService_1930889e5b6c51cbe57428f9a664e4dc)]

... 34 more

2018-11-09 04:47:19,680 DEBUG org.apache.flink.runtime.rest.handler.legacy.metrics.MetricFetcher - Could not retrieve QueryServiceGateway.

java.util.concurrent.CompletionException: akka.actor.ActorNotFound: Actor not found for: ActorSelection[Anchor(akka.tcp://flink-metrics@fps-flink-taskmanager-6f8f687fc8-bzq2w:39393/), Path(/user/MetricQueryService_fcaf62301df0a6aeb29a65470cfe1e7a)]