2021-02-28 13:14:07 Full thread dump OpenJDK 64-Bit Server VM (11.0.8+10 mixed mode): Threads class SMR info: _java_thread_list=0x00007f78080217b0, length=44, elements={ 0x00007f784c1ac000, 0x00007f784c1ae800, 0x00007f784c1ca800, 0x00007f784c1cc800, 0x00007f784c1d1000, 0x00007f784c1d3000, 0x00007f784c1d5000, 0x00007f784c23f800, 0x00007f784c253000, 0x00007f784d029000, 0x00007f784c880000, 0x00007f784e99e800, 0x00007f784eacb000, 0x00007f77e4010800, 0x00007f77e401c800, 0x00007f77e401e800, 0x00007f77d402f800, 0x00007f77d4040000, 0x00007f77d4043000, 0x00007f77d407b000, 0x00007f784e6b7000, 0x0000555f31037000, 0x0000555f31039000, 0x00007f77b400b800, 0x00007f77b4020000, 0x00007f77b4024800, 0x00007f77b4039800, 0x00007f784e6e1800, 0x00007f784e6e4000, 0x00007f784c71e800, 0x00007f784c720800, 0x00007f77a48c1000, 0x00007f77a48ea800, 0x00007f784c01a000, 0x00007f77c4002000, 0x00007f77d8007000, 0x00007f77e407a000, 0x00007f77e40a5800, 0x00007f7824101800, 0x00007f782411f000, 0x00007f7824241800, 0x00007f782443e800, 0x00007f77dc019000, 0x00007f7808027800 } "Reference Handler" #2 daemon prio=10 os_prio=0 cpu=10.95ms elapsed=3664.24s tid=0x00007f784c1ac000 nid=0x123 waiting on condition [0x00007f783436e000] java.lang.Thread.State: RUNNABLE at java.lang.ref.Reference.waitForReferencePendingList(java.base@11.0.8/Native Method) at java.lang.ref.Reference.processPendingReferences(java.base@11.0.8/Unknown Source) at java.lang.ref.Reference$ReferenceHandler.run(java.base@11.0.8/Unknown Source) "Finalizer" #3 daemon prio=8 os_prio=0 cpu=6.80ms elapsed=3664.24s tid=0x00007f784c1ae800 nid=0x124 in Object.wait() [0x00007f783426d000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.8/Native Method) - waiting on <0x00000000f19199b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(java.base@11.0.8/Unknown Source) - waiting to re-lock in wait() <0x00000000f19199b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(java.base@11.0.8/Unknown Source) at java.lang.ref.Finalizer$FinalizerThread.run(java.base@11.0.8/Unknown Source) "Signal Dispatcher" #4 daemon prio=9 os_prio=0 cpu=0.18ms elapsed=3664.21s tid=0x00007f784c1ca800 nid=0x125 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Attach Listener" #5 daemon prio=5 os_prio=0 cpu=16.47ms elapsed=3664.21s tid=0x00007f784c1cc800 nid=0x126 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 cpu=20067.36ms elapsed=3664.21s tid=0x00007f784c1d1000 nid=0x127 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE No compile task "C1 CompilerThread0" #7 daemon prio=9 os_prio=0 cpu=6592.50ms elapsed=3664.21s tid=0x00007f784c1d3000 nid=0x128 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE No compile task "Sweeper thread" #8 daemon prio=9 os_prio=0 cpu=1370.62ms elapsed=3664.21s tid=0x00007f784c1d5000 nid=0x129 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Service Thread" #9 daemon prio=9 os_prio=0 cpu=0.33ms elapsed=3664.11s tid=0x00007f784c23f800 nid=0x12a runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Common-Cleaner" #10 daemon prio=8 os_prio=0 cpu=18.09ms elapsed=3664.07s tid=0x00007f784c253000 nid=0x12c in Object.wait() [0x00007f781f084000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.8/Native Method) - waiting on at java.lang.ref.ReferenceQueue.remove(java.base@11.0.8/Unknown Source) - waiting to re-lock in wait() <0x00000000f1960528> (a java.lang.ref.ReferenceQueue$Lock) at jdk.internal.ref.CleanerImpl.run(java.base@11.0.8/Unknown Source) at java.lang.Thread.run(java.base@11.0.8/Unknown Source) at jdk.internal.misc.InnocuousThread.run(java.base@11.0.8/Unknown Source) "kafka-producer-network-thread | producer-1" #13 daemon prio=5 os_prio=0 cpu=1134.66ms elapsed=3660.35s tid=0x00007f784d029000 nid=0x12d runnable [0x00007f781dd2b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.8/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.8/Unknown Source) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.8/Unknown Source) - locked <0x00000000f1f65a88> (a sun.nio.ch.Util$2) - locked <0x00000000f1f65a30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.8/Unknown Source) at org.apache.kafka.common.network.Selector.select(Selector.java:873) at org.apache.kafka.common.network.Selector.poll(Selector.java:469) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:544) at org.apache.kafka.clients.producer.internals.Sender.runOnce(Sender.java:325) at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:240) at java.lang.Thread.run(java.base@11.0.8/Unknown Source) "kafka-producer-network-thread | producer-2" #16 daemon prio=5 os_prio=0 cpu=3627.38ms elapsed=3657.39s tid=0x00007f784c880000 nid=0x12e runnable [0x00007f781d227000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.8/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.8/Unknown Source) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.8/Unknown Source) - locked <0x00000000f2061400> (a sun.nio.ch.Util$2) - locked <0x00000000f204cc68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.8/Unknown Source) at org.apache.kafka.common.network.Selector.select(Selector.java:873) at org.apache.kafka.common.network.Selector.poll(Selector.java:469) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:544) at org.apache.kafka.clients.producer.internals.Sender.runOnce(Sender.java:325) at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:240) at java.lang.Thread.run(java.base@11.0.8/Unknown Source) "flink-scheduler-1" #17 prio=5 os_prio=0 cpu=47373.37ms elapsed=3655.23s tid=0x00007f784e99e800 nid=0x12f waiting on condition [0x00007f781cb26000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.8/Native Method) at akka.actor.LightArrayRevolverScheduler.waitNanos(LightArrayRevolverScheduler.scala:85) at akka.actor.LightArrayRevolverScheduler$$anon$3.nextTick(LightArrayRevolverScheduler.scala:265) at akka.actor.LightArrayRevolverScheduler$$anon$3.run(LightArrayRevolverScheduler.scala:235) at java.lang.Thread.run(java.base@11.0.8/Unknown Source) "flink-akka.actor.default-dispatcher-2" #18 prio=5 os_prio=0 cpu=1377.29ms elapsed=3654.56s tid=0x00007f784eacb000 nid=0x130 waiting on condition [0x00007f781c825000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.8/Native Method) - parking to wait for <0x00000000f20fb600> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at sun.misc.Unsafe.park(jdk.unsupported@11.0.8/Unknown Source) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) "flink-akka.actor.default-dispatcher-4" #20 prio=5 os_prio=0 cpu=376.38ms elapsed=3654.52s tid=0x00007f77e4010800 nid=0x132 waiting on condition [0x00007f781c623000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.8/Native Method) - parking to wait for <0x00000000f20fb600> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at sun.misc.Unsafe.park(jdk.unsupported@11.0.8/Unknown Source) at akka.dispatch.forkjoin.ForkJoinPool.idleAwaitWork(ForkJoinPool.java:2135) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2067) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) "flink-akka.remote.default-remote-dispatcher-5" #21 prio=5 os_prio=0 cpu=4226.30ms elapsed=3654.39s tid=0x00007f77e401c800 nid=0x133 waiting on condition [0x00007f781c322000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.8/Native Method) - parking to wait for <0x00000000f22bd130> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at sun.misc.Unsafe.park(jdk.unsupported@11.0.8/Unknown Source) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) "flink-akka.remote.default-remote-dispatcher-6" #22 prio=5 os_prio=0 cpu=4695.06ms elapsed=3654.37s tid=0x00007f77e401e800 nid=0x134 runnable [0x00007f781c221000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.8/Native Method) - parking to wait for <0x00000000f22bd130> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at sun.misc.Unsafe.park(jdk.unsupported@11.0.8/Unknown Source) at akka.dispatch.forkjoin.ForkJoinPool.idleAwaitWork(ForkJoinPool.java:2135) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2067) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) "New I/O worker #1" #23 prio=5 os_prio=0 cpu=1302.84ms elapsed=3654.11s tid=0x00007f77d402f800 nid=0x135 runnable [0x00007f781c120000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.8/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.8/Unknown Source) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.8/Unknown Source) - locked <0x00000000f2133aa8> (a sun.nio.ch.Util$2) - locked <0x00000000f2133958> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.8/Unknown Source) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.apache.flink.shaded.akka.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.apache.flink.shaded.akka.org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.8/Unknown Source) at java.lang.Thread.run(java.base@11.0.8/Unknown Source) "New I/O boss #2" #25 prio=5 os_prio=0 cpu=941.47ms elapsed=3654.08s tid=0x00007f77d4040000 nid=0x136 runnable [0x00007f77cfffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.8/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.8/Unknown Source) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.8/Unknown Source) - locked <0x00000000f212f820> (a sun.nio.ch.Util$2) - locked <0x00000000f212f6d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.8/Unknown Source) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.NioClientBoss.run(NioClientBoss.java:42) at org.apache.flink.shaded.akka.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.apache.flink.shaded.akka.org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.8/Unknown Source) at java.lang.Thread.run(java.base@11.0.8/Unknown Source) "New I/O worker #3" #26 prio=5 os_prio=0 cpu=885.67ms elapsed=3654.06s tid=0x00007f77d4043000 nid=0x137 runnable [0x00007f77cfefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.8/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.8/Unknown Source) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.8/Unknown Source) - locked <0x00000000f225c8e0> (a sun.nio.ch.Util$2) - locked <0x00000000f225c790> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.8/Unknown Source) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.apache.flink.shaded.akka.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.apache.flink.shaded.akka.org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.8/Unknown Source) at java.lang.Thread.run(java.base@11.0.8/Unknown Source) "New I/O server boss #4" #27 prio=5 os_prio=0 cpu=75.10ms elapsed=3654.05s tid=0x00007f77d407b000 nid=0x138 runnable [0x00007f77cfdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.8/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.8/Unknown Source) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.8/Unknown Source) - locked <0x00000000f225d5a0> (a sun.nio.ch.Util$2) - locked <0x00000000f225d1a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.8/Unknown Source) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.NioServerBoss.select(NioServerBoss.java:163) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.NioServerBoss.run(NioServerBoss.java:42) at org.apache.flink.shaded.akka.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.apache.flink.shaded.akka.org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.8/Unknown Source) at java.lang.Thread.run(java.base@11.0.8/Unknown Source) "flink-metrics-scheduler-1" #29 prio=5 os_prio=0 cpu=46547.31ms elapsed=3653.17s tid=0x00007f784e6b7000 nid=0x13a waiting on condition [0x00007f77cf7fa000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.8/Native Method) at akka.actor.LightArrayRevolverScheduler.waitNanos(LightArrayRevolverScheduler.scala:85) at akka.actor.LightArrayRevolverScheduler$$anon$3.nextTick(LightArrayRevolverScheduler.scala:265) at akka.actor.LightArrayRevolverScheduler$$anon$3.run(LightArrayRevolverScheduler.scala:235) at java.lang.Thread.run(java.base@11.0.8/Unknown Source) "flink-metrics-akka.remote.default-remote-dispatcher-3" #31 prio=1 os_prio=0 cpu=1941.11ms elapsed=3653.11s tid=0x0000555f31037000 nid=0x13c waiting on condition [0x00007f77cf5f8000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.8/Native Method) - parking to wait for <0x00000000f25054f0> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at sun.misc.Unsafe.park(jdk.unsupported@11.0.8/Unknown Source) at akka.dispatch.forkjoin.ForkJoinPool.idleAwaitWork(ForkJoinPool.java:2135) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2067) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) "flink-metrics-akka.remote.default-remote-dispatcher-4" #32 prio=1 os_prio=0 cpu=2118.78ms elapsed=3653.10s tid=0x0000555f31039000 nid=0x13d waiting on condition [0x00007f77cf4f7000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.8/Native Method) - parking to wait for <0x00000000f25054f0> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at sun.misc.Unsafe.park(jdk.unsupported@11.0.8/Unknown Source) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) "New I/O worker #5" #33 prio=1 os_prio=0 cpu=776.31ms elapsed=3653.10s tid=0x00007f77b400b800 nid=0x13e runnable [0x00007f77cf3f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.8/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.8/Unknown Source) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.8/Unknown Source) - locked <0x00000000f24efa40> (a sun.nio.ch.Util$2) - locked <0x00000000f24ef8f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.8/Unknown Source) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.apache.flink.shaded.akka.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.apache.flink.shaded.akka.org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.8/Unknown Source) at java.lang.Thread.run(java.base@11.0.8/Unknown Source) "New I/O boss #6" #35 prio=1 os_prio=0 cpu=945.58ms elapsed=3653.09s tid=0x00007f77b4020000 nid=0x13f runnable [0x00007f77cf2f5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.8/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.8/Unknown Source) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.8/Unknown Source) - locked <0x00000000f24f3c80> (a sun.nio.ch.Util$2) - locked <0x00000000f24f3b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.8/Unknown Source) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.NioClientBoss.run(NioClientBoss.java:42) at org.apache.flink.shaded.akka.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.apache.flink.shaded.akka.org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.8/Unknown Source) at java.lang.Thread.run(java.base@11.0.8/Unknown Source) "New I/O worker #7" #36 prio=1 os_prio=0 cpu=825.57ms elapsed=3653.09s tid=0x00007f77b4024800 nid=0x140 runnable [0x00007f77cf1f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.8/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.8/Unknown Source) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.8/Unknown Source) - locked <0x00000000f24f4500> (a sun.nio.ch.Util$2) - locked <0x00000000f24f43b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.8/Unknown Source) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.apache.flink.shaded.akka.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.apache.flink.shaded.akka.org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.8/Unknown Source) at java.lang.Thread.run(java.base@11.0.8/Unknown Source) "New I/O server boss #8" #37 prio=1 os_prio=0 cpu=6.69ms elapsed=3653.08s tid=0x00007f77b4039800 nid=0x141 runnable [0x00007f77cf0f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.8/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.8/Unknown Source) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.8/Unknown Source) - locked <0x00000000f24f4c10> (a sun.nio.ch.Util$2) - locked <0x00000000f24f4ac0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.8/Unknown Source) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.NioServerBoss.select(NioServerBoss.java:163) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) at org.apache.flink.shaded.akka.org.jboss.netty.channel.socket.nio.NioServerBoss.run(NioServerBoss.java:42) at org.apache.flink.shaded.akka.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.apache.flink.shaded.akka.org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.8/Unknown Source) at java.lang.Thread.run(java.base@11.0.8/Unknown Source) "Timer-0" #41 daemon prio=5 os_prio=0 cpu=8.40ms elapsed=3652.84s tid=0x00007f784e6e1800 nid=0x144 in Object.wait() [0x00007f77ceef1000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.8/Native Method) - waiting on at java.util.TimerThread.mainLoop(java.base@11.0.8/Unknown Source) - waiting to re-lock in wait() <0x00000000f24c2a80> (a java.util.TaskQueue) at java.util.TimerThread.run(java.base@11.0.8/Unknown Source) "Timer-1" #43 daemon prio=5 os_prio=0 cpu=4.95ms elapsed=3652.81s tid=0x00007f784e6e4000 nid=0x145 in Object.wait() [0x00007f77cedf0000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.8/Native Method) - waiting on at java.util.TimerThread.mainLoop(java.base@11.0.8/Unknown Source) - waiting to re-lock in wait() <0x00000000f24c3a48> (a java.util.TaskQueue) at java.util.TimerThread.run(java.base@11.0.8/Unknown Source) "IOManager writer thread #1" #45 daemon prio=5 os_prio=0 cpu=0.20ms elapsed=3652.44s tid=0x00007f784c71e800 nid=0x146 waiting on condition [0x00007f77cecef000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.8/Native Method) - parking to wait for <0x00000000f24b5a30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.8/Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.8/Unknown Source) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.8/Unknown Source) at org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync$WriterThread.run(IOManagerAsync.java:460) "IOManager reader thread #1" #46 daemon prio=5 os_prio=0 cpu=0.24ms elapsed=3652.43s tid=0x00007f784c720800 nid=0x147 waiting on condition [0x00007f77cebee000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.8/Native Method) - parking to wait for <0x00000000f24b5fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.8/Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.8/Unknown Source) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.8/Unknown Source) at org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync$ReaderThread.run(IOManagerAsync.java:354) "Flink Netty Server (0) Thread 0" #49 daemon prio=5 os_prio=0 cpu=630.33ms elapsed=3651.41s tid=0x00007f77a48c1000 nid=0x148 runnable [0x00007f77ce6dd000] java.lang.Thread.State: RUNNABLE at org.apache.flink.shaded.netty4.io.netty.channel.epoll.Native.epollWait0(Native Method) at org.apache.flink.shaded.netty4.io.netty.channel.epoll.Native.epollWait(Native.java:96) at org.apache.flink.shaded.netty4.io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:276) at org.apache.flink.shaded.netty4.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:305) at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:918) at org.apache.flink.shaded.netty4.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at java.lang.Thread.run(java.base@11.0.8/Unknown Source) "Memory Logger" #52 daemon prio=1 os_prio=0 cpu=2735.65ms elapsed=3651.22s tid=0x00007f77a48ea800 nid=0x14a waiting on condition [0x00007f77ce5dc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.8/Native Method) at org.apache.flink.runtime.taskmanager.MemoryLogger.run(MemoryLogger.java:135) "DestroyJavaVM" #53 prio=5 os_prio=0 cpu=6060.87ms elapsed=3651.22s tid=0x00007f784c01a000 nid=0x11c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Hashed wheel timer #1" #24 prio=5 os_prio=0 cpu=5252.55ms elapsed=3650.93s tid=0x00007f77c4002000 nid=0x14b waiting on condition [0x00007f77ce2db000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.8/Native Method) at org.apache.flink.shaded.akka.org.jboss.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:445) at org.apache.flink.shaded.akka.org.jboss.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:364) at org.apache.flink.shaded.akka.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at java.lang.Thread.run(java.base@11.0.8/Unknown Source) "flink-akka.actor.default-dispatcher-13" #56 prio=5 os_prio=0 cpu=237.04ms elapsed=3648.36s tid=0x00007f77d8007000 nid=0x14d waiting on condition [0x00007f77ce1da000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.8/Native Method) - parking to wait for <0x00000000f20fb600> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool) at sun.misc.Unsafe.park(jdk.unsupported@11.0.8/Unknown Source) at akka.dispatch.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) "pool-4-thread-1" #57 prio=5 os_prio=0 cpu=2.08ms elapsed=2202.65s tid=0x00007f77e407a000 nid=0x214 waiting on condition [0x00007f77cfafb000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.8/Native Method) - parking to wait for <0x00000000f23247a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.8/Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.8/Unknown Source) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.8/Unknown Source) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.8/Unknown Source) at java.lang.Thread.run(java.base@11.0.8/Unknown Source) "Flink-MetricRegistry-thread-1" #61 daemon prio=5 os_prio=0 cpu=116.66ms elapsed=2199.99s tid=0x00007f77e40a5800 nid=0x218 waiting on condition [0x00007f781c724000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.8/Native Method) - parking to wait for <0x00000000f252d840> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.8/Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.8/Unknown Source) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.8/Unknown Source) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.8/Unknown Source) at java.lang.Thread.run(java.base@11.0.8/Unknown Source) "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" #67 daemon prio=5 os_prio=0 cpu=1.02ms elapsed=2194.06s tid=0x00007f7824101800 nid=0x21f in Object.wait() [0x00007f77cd3b4000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.8/Native Method) - waiting on <0x00000000f272c0e0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(java.base@11.0.8/Unknown Source) - waiting to re-lock in wait() <0x00000000f272c0e0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(java.base@11.0.8/Unknown Source) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3806) at java.lang.Thread.run(java.base@11.0.8/Unknown Source) "MutableQuantiles-0" #68 daemon prio=5 os_prio=0 cpu=765.59ms elapsed=2193.81s tid=0x00007f782411f000 nid=0x220 waiting on condition [0x00007f77cd2b3000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.8/Native Method) - parking to wait for <0x00000000f294a0d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.8/Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.8/Unknown Source) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.8/Unknown Source) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.8/Unknown Source) at java.lang.Thread.run(java.base@11.0.8/Unknown Source) "Timer for 's3a-file-system' metrics system" #69 daemon prio=5 os_prio=0 cpu=51.29ms elapsed=2189.95s tid=0x00007f7824241800 nid=0x221 in Object.wait() [0x00007f77ccfb2000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.8/Native Method) - waiting on at java.util.TimerThread.mainLoop(java.base@11.0.8/Unknown Source) - waiting to re-lock in wait() <0x00000000f2945cd8> (a java.util.TaskQueue) at java.util.TimerThread.run(java.base@11.0.8/Unknown Source) "java-sdk-http-connection-reaper" #70 daemon prio=5 os_prio=0 cpu=128.00ms elapsed=2184.82s tid=0x00007f782443e800 nid=0x222 waiting on condition [0x00007f77cc280000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.8/Native Method) at com.amazonaws.http.IdleConnectionReaper.run(IdleConnectionReaper.java:188) "flink-taskexecutor-io-thread-1" #72 daemon prio=5 os_prio=0 cpu=0.25ms elapsed=2177.26s tid=0x00007f77dc019000 nid=0x230 waiting on condition [0x00007f77cd6d7000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.8/Native Method) - parking to wait for <0x00000000f24b21b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.8/Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.8/Unknown Source) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.8/Unknown Source) at java.lang.Thread.run(java.base@11.0.8/Unknown Source) "flink-taskexecutor-io-thread-2" #78 daemon prio=5 os_prio=0 cpu=0.82ms elapsed=2147.65s tid=0x00007f7808027800 nid=0x236 waiting on condition [0x00007f77cdcd9000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.8/Native Method) - parking to wait for <0x00000000f24b21b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.8/Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.8/Unknown Source) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.8/Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.8/Unknown Source) at java.lang.Thread.run(java.base@11.0.8/Unknown Source) "VM Thread" os_prio=0 cpu=2525.37ms elapsed=3664.26s tid=0x00007f784c1a1800 nid=0x122 runnable "GC Thread#0" os_prio=0 cpu=3053.20ms elapsed=3664.30s tid=0x00007f784c033000 nid=0x11d runnable "G1 Main Marker" os_prio=0 cpu=4.27ms elapsed=3664.30s tid=0x00007f784c03e000 nid=0x11e runnable "G1 Conc#0" os_prio=0 cpu=38.76ms elapsed=3664.30s tid=0x00007f784c040000 nid=0x11f runnable "G1 Refine#0" os_prio=0 cpu=12.58ms elapsed=3664.30s tid=0x00007f784c107000 nid=0x120 runnable "G1 Young RemSet Sampling" os_prio=0 cpu=1371.16ms elapsed=3664.30s tid=0x00007f784c109000 nid=0x121 runnable "VM Periodic Task Thread" os_prio=0 cpu=7641.69ms elapsed=3664.11s tid=0x00007f784c242000 nid=0x12b waiting on condition JNI global refs: 28, weak refs: 0