(DEPRECATED) Apache Flink User Mailing List archive.
Login  Register

DataStreamCalcRule$1802" grows beyond 64 KB when execute long sql.

classic Classic list List threaded Threaded
♦
♦
 5 messages Options Options
 
Embed post
Permalink
zhangminglei
Reply | Threaded
Open this post in threaded view
♦
♦
| More
Print post
Permalink
Jun 19, 2018; 2:27pm

DataStreamCalcRule$1802" grows beyond 64 KB when execute long sql.

zhangminglei
50 posts
Hi, friends.

When I execute a long sql and get the follow error, how can I have a quick fix ?

org.apache.flink.api.common.InvalidProgramException: Table program cannot be compiled. This is a bug. Please file an issue.
at org.apache.flink.table.codegen.Compiler$class.compile(Compiler.scala:36)
at org.apache.flink.table.runtime.CRowProcessRunner.compile(CRowProcessRunner.scala:35)
at org.apache.flink.table.runtime.CRowProcessRunner.open(CRowProcessRunner.scala:49)
at org.apache.flink.api.common.functions.util.FunctionUtils.openFunction(FunctionUtils.java:36)
at org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator.open(AbstractUdfStreamOperator.java:102)
at org.apache.flink.streaming.api.operators.ProcessOperator.open(ProcessOperator.java:56)
at org.apache.flink.streaming.runtime.tasks.StreamTask.openAllOperators(StreamTask.java:393)
at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:254)
at org.apache.flink.runtime.taskmanager.Task.run(Task.java:718)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: Compiling "DataStreamCalcRule$1802": Code of method "processElement(Ljava/lang/Object;Lorg/apache/flink/streaming/api/functions/ProcessFunction$Context;Lorg/apache/flink/util/Collector;)V" of class "DataStreamCalcRule$1802" grows beyond 64 KB
at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:361)

select case when cast(status as bigint) in (200) then 10 else 1 end as pv,\
case when cast(status as bigint) between 303 and 1000 then 1 when cast(status as bigint) between 100 and 199 then 1 else 0 end as service_fail,\
case when cast(status as bigint) in (200) then 10 when cast(status as bigint) in (301,302) then 1 when cast(status as bigint) between 201 and 299 then 1 else 0 end as service_success,\
case when cast(status as bigint) not between 100 and 1000 then 1 else 0 end as network_fail,\
qqiplib(ip, 'isp') as isp,\
case when response_time - request_time <= 60000 then response_time - request_time else 0 end as response_t,\
case when response_time - request_time <= 60000 then 1 else 0 end as count_in,\
case when host in ('116.31.114.22','116.31.114.23','183.60.219.231','183.60.219.232','183.60.219.235','183.60.219.236','183.60.220.231','183.60.220.232',\
'183.60.219.247','183.60.219.248','183.60.219.243','183.60.219.244','183.60.219.251','183.60.219.252','116.31.114.202','116.31.114.204','116.31.114.206'\
,'116.31.114.208') then '佛山力通电信_SR' \
when host in ('183.232.169.11','183.232.169.12','183.232.169.13','183.232.169.14','183.232.169.15','183.232.169.16','183.232.169.17','183.232.169.18') \
then '佛山力通移动_SR' \
when host in ('112.93.112.11','112.93.112.12','112.93.112.13','112.93.112.14','112.93.112.15','112.93.112.16','112.93.112.17','112.93.112.18') \
then '佛山力通联通_SR' \
when host in ('114.67.56.79','114.67.56.80','114.67.56.83','114.67.56.84','114.67.56.87','114.67.56.88','114.67.56.112','114.67.56.113','114.67.56.116',\
'114.67.56.117','114.67.60.214','114.67.60.215','114.67.54.111','114.67.54.112','114.67.56.95','114.67.56.96','114.67.54.12','114.67.54.13',\
'114.67.56.93') \
then '佛山力通BGP_SR' \
when host in ('114.67.56.94','114.67.56.102','114.67.56.103','114.67.56.106','114.67.56.107','183.60.220.231','183.60.220.232','183.60.219.247',\
'183.60.219.248','114.67.60.201','114.67.60.203','114.67.60.205','114.67.60.207') \
then '佛山力通BGP_SR' \
when host in ('183.240.167.24','183.240.167.25','183.240.167.26','183.240.167.27','183.240.167.28','183.240.167.29','183.240.167.30','183.240.167.31') \
then '佛山互联移动_SR' when host in ('27.45.160.24','27.45.160.25','27.45.160.26','27.45.160.27','27.45.160.28','27.45.160.29','27.45.160.30','27.45.160.31') \
then '佛山互联联通_SR' \
when host in ('43.255.228.11','43.255.228.12','43.255.228.13','43.255.228.14','43.255.228.15','43.255.228.16','43.255.228.17','43.255.228.18','43.255.228.19',\
'43.255.228.20','43.255.228.21','43.255.228.22','43.255.228.23') \
then '佛山互联BGP_SR' \
when host in ('43.255.228.24','43.255.228.25','43.255.228.26','43.255.228.27','43.255.228.28','43.255.228.29','43.255.228.30','43.255.228.31','43.255.228.32',\
'43.255.228.33','43.255.228.34') \
then '佛山互联BGP_SR' \
when host in ('14.17.91.113','14.17.91.114','14.17.72.11','14.17.72.12','14.17.72.15','14.17.72.16','14.17.72.19','14.17.72.20','183.61.170.121','183.61.170.122',\
'14.17.91.120','14.17.91.121','14.17.72.79','14.17.72.80','14.17.72.81','14.17.72.82') \
then '东莞电信_SR' \
when host in ('221.228.213.94','221.228.213.95','221.228.213.68','221.228.213.69','221.228.213.76','221.228.213.77','221.228.213.98','221.228.213.99',\
'221.228.213.122','221.228.213.124') \
then '无锡电信_SR' \
when host in ('43.247.88.49','43.247.88.50','43.247.88.135','43.247.88.136','43.247.88.16','43.247.88.17','43.247.88.143','43.247.88.144','43.247.88.183','43.247.88.185',\
'43.247.88.187','43.247.88.189') \
then '无锡BGP_SR' \
when host in ('101.251.216.140','101.251.216.141','101.251.216.157','101.251.216.158') \
then '北京BGP_SR' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('116.31.114.22','116.31.114.23','183.60.219.231','183.60.219.232','183.60.219.235','183.60.219.236','183.60.220.231',\
'183.60.220.232','183.60.219.247','183.60.219.248','183.60.219.243','183.60.219.244','183.60.219.251','183.60.219.252','116.31.114.202','116.31.114.204',\
'116.31.114.206','116.31.114.208') \
then '佛山力通电信_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('183.232.169.11','183.232.169.12','183.232.169.13','183.232.169.14','183.232.169.15','183.232.169.16',\
'183.232.169.17','183.232.169.18') \
then '佛山力通移动_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('112.93.112.11','112.93.112.12','112.93.112.13','112.93.112.14','112.93.112.15','112.93.112.16','112.93.112.17','112.93.112.18') \
then '佛山力通联通_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('114.67.56.79','114.67.56.80','114.67.56.83','114.67.56.84','114.67.56.87','114.67.56.88','114.67.56.112',\
'114.67.56.113','114.67.56.116','114.67.56.117','114.67.60.214','114.67.60.215','114.67.54.111') \
then '佛山力通BGP_GSLB' \
when host in ('114.67.54.112','114.67.56.95','114.67.56.96','114.67.54.12','114.67.54.13','114.67.56.93','114.67.56.94','114.67.56.102','114.67.56.103','114.67.56.106',\
'114.67.56.107','183.60.220.231','183.60.220.232','183.60.219.247','183.60.219.248','114.67.60.201','114.67.60.203','114.67.60.205','114.67.60.207') \
then '佛山力通BGP_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('183.240.167.24','183.240.167.25','183.240.167.26','183.240.167.27','183.240.167.28','183.240.167.29',\
'183.240.167.30','183.240.167.31') \
then '佛山互联移动_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('43.255.228.11','43.255.228.12','43.255.228.13','43.255.228.14','43.255.228.15','43.255.228.16',\
'43.255.228.17') \
then '佛山互联BGP_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('43.255.228.18','43.255.228.19','43.255.228.20') \
then '佛山互联BGP_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('43.255.228.21') \
then '佛山互联BGP_GSLB' else '其它' end as access_type from dw_log_app_api_monitor_ds
Thanks
Zhangminglei
Fabian Hueske-2
Reply | Threaded
Open this post in threaded view
♦
♦
| More
Print post
Permalink
Jun 19, 2018; 2:36pm

Re: DataStreamCalcRule$1802" grows beyond 64 KB when execute long sql.

Fabian Hueske-2
1749 posts
Hi,

Which version are you using? We fixed a similar issue for Flink 1.5.0.
If you can't upgrade yet, you can also implement a user-defined function that evaluates the big CASE WHEN statement.

Best, Fabian

2018-06-19 16:27 GMT+02:00 zhangminglei <[hidden email]>:
Hi, friends.

When I execute a long sql and get the follow error, how can I have a quick fix ?

org.apache.flink.api.common.InvalidProgramException: Table program cannot be compiled. This is a bug. Please file an issue.
        at org.apache.flink.table.codegen.Compiler$class.compile(Compiler.scala:36)
        at org.apache.flink.table.runtime.CRowProcessRunner.compile(CRowProcessRunner.scala:35)
        at org.apache.flink.table.runtime.CRowProcessRunner.open(CRowProcessRunner.scala:49)
        at org.apache.flink.api.common.functions.util.FunctionUtils.openFunction(FunctionUtils.java:36)
        at org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator.open(AbstractUdfStreamOperator.java:102)
        at org.apache.flink.streaming.api.operators.ProcessOperator.open(ProcessOperator.java:56)
        at org.apache.flink.streaming.runtime.tasks.StreamTask.openAllOperators(StreamTask.java:393)
        at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:254)
        at org.apache.flink.runtime.taskmanager.Task.run(Task.java:718)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: Compiling "DataStreamCalcRule$1802": Code of method "processElement(Ljava/lang/Object;Lorg/apache/flink/streaming/api/functions/ProcessFunction$Context;Lorg/apache/flink/util/Collector;)V" of class "DataStreamCalcRule$1802" grows beyond 64 KB
        at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:361)

select case when cast(status as bigint) in (200) then 10 else 1 end as pv,\
case when cast(status as bigint) between 303 and 1000 then 1 when cast(status as bigint) between 100 and 199 then 1 else 0 end as service_fail,\
case when cast(status as bigint) in (200) then 10 when cast(status as bigint) in (301,302) then 1 when cast(status as bigint) between 201 and 299 then 1 else 0 end as service_success,\
case when cast(status as bigint) not between 100 and 1000 then 1 else 0 end as network_fail,\
qqiplib(ip, 'isp') as isp,\
case when response_time - request_time <= 60000 then response_time - request_time else 0 end as response_t,\
case when response_time - request_time <= 60000 then 1 else 0 end as count_in,\
case when host in ('116.31.114.22','116.31.114.23','183.60.219.231','183.60.219.232','183.60.219.235','183.60.219.236','183.60.220.231','183.60.220.232',\
'183.60.219.247','183.60.219.248','183.60.219.243','183.60.219.244','183.60.219.251','183.60.219.252','116.31.114.202','116.31.114.204','116.31.114.206'\
,'116.31.114.208') then '佛山力通电信_SR' \
when host in ('183.232.169.11','183.232.169.12','183.232.169.13','183.232.169.14','183.232.169.15','183.232.169.16','183.232.169.17','183.232.169.18') \
then '佛山力通移动_SR' \
when host in ('112.93.112.11','112.93.112.12','112.93.112.13','112.93.112.14','112.93.112.15','112.93.112.16','112.93.112.17','112.93.112.18') \
then '佛山力通联通_SR' \
when host in ('114.67.56.79','114.67.56.80','114.67.56.83','114.67.56.84','114.67.56.87','114.67.56.88','114.67.56.112','114.67.56.113','114.67.56.116',\
'114.67.56.117','114.67.60.214','114.67.60.215','114.67.54.111','114.67.54.112','114.67.56.95','114.67.56.96','114.67.54.12','114.67.54.13',\
'114.67.56.93') \
then '佛山力通BGP_SR' \
when host in ('114.67.56.94','114.67.56.102','114.67.56.103','114.67.56.106','114.67.56.107','183.60.220.231','183.60.220.232','183.60.219.247',\
'183.60.219.248','114.67.60.201','114.67.60.203','114.67.60.205','114.67.60.207') \
then '佛山力通BGP_SR' \
when host in ('183.240.167.24','183.240.167.25','183.240.167.26','183.240.167.27','183.240.167.28','183.240.167.29','183.240.167.30','183.240.167.31') \
then '佛山互联移动_SR' when host in ('27.45.160.24','27.45.160.25','27.45.160.26','27.45.160.27','27.45.160.28','27.45.160.29','27.45.160.30','27.45.160.31') \
then '佛山互联联通_SR' \
when host in ('43.255.228.11','43.255.228.12','43.255.228.13','43.255.228.14','43.255.228.15','43.255.228.16','43.255.228.17','43.255.228.18','43.255.228.19',\
'43.255.228.20','43.255.228.21','43.255.228.22','43.255.228.23') \
then '佛山互联BGP_SR' \
when host in ('43.255.228.24','43.255.228.25','43.255.228.26','43.255.228.27','43.255.228.28','43.255.228.29','43.255.228.30','43.255.228.31','43.255.228.32',\
'43.255.228.33','43.255.228.34') \
then '佛山互联BGP_SR' \
when host in ('14.17.91.113','14.17.91.114','14.17.72.11','14.17.72.12','14.17.72.15','14.17.72.16','14.17.72.19','14.17.72.20','183.61.170.121','183.61.170.122',\
'14.17.91.120','14.17.91.121','14.17.72.79','14.17.72.80','14.17.72.81','14.17.72.82') \
then '东莞电信_SR' \
when host in ('221.228.213.94','221.228.213.95','221.228.213.68','221.228.213.69','221.228.213.76','221.228.213.77','221.228.213.98','221.228.213.99',\
'221.228.213.122','221.228.213.124') \
then '无锡电信_SR' \
when host in ('43.247.88.49','43.247.88.50','43.247.88.135','43.247.88.136','43.247.88.16','43.247.88.17','43.247.88.143','43.247.88.144','43.247.88.183','43.247.88.185',\
'43.247.88.187','43.247.88.189') \
then '无锡BGP_SR' \
when host in ('101.251.216.140','101.251.216.141','101.251.216.157','101.251.216.158') \
then '北京BGP_SR' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('116.31.114.22','116.31.114.23','183.60.219.231','183.60.219.232','183.60.219.235','183.60.219.236','183.60.220.231',\
'183.60.220.232','183.60.219.247','183.60.219.248','183.60.219.243','183.60.219.244','183.60.219.251','183.60.219.252','116.31.114.202','116.31.114.204',\
'116.31.114.206','116.31.114.208') \
then '佛山力通电信_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('183.232.169.11','183.232.169.12','183.232.169.13','183.232.169.14','183.232.169.15','183.232.169.16',\
'183.232.169.17','183.232.169.18') \
then '佛山力通移动_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('112.93.112.11','112.93.112.12','112.93.112.13','112.93.112.14','112.93.112.15','112.93.112.16','112.93.112.17','112.93.112.18') \
then '佛山力通联通_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('114.67.56.79','114.67.56.80','114.67.56.83','114.67.56.84','114.67.56.87','114.67.56.88','114.67.56.112',\
'114.67.56.113','114.67.56.116','114.67.56.117','114.67.60.214','114.67.60.215','114.67.54.111') \
then '佛山力通BGP_GSLB' \
when host in ('114.67.54.112','114.67.56.95','114.67.56.96','114.67.54.12','114.67.54.13','114.67.56.93','114.67.56.94','114.67.56.102','114.67.56.103','114.67.56.106',\
'114.67.56.107','183.60.220.231','183.60.220.232','183.60.219.247','183.60.219.248','114.67.60.201','114.67.60.203','114.67.60.205','114.67.60.207') \
then '佛山力通BGP_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('183.240.167.24','183.240.167.25','183.240.167.26','183.240.167.27','183.240.167.28','183.240.167.29',\
'183.240.167.30','183.240.167.31') \
then '佛山互联移动_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('43.255.228.11','43.255.228.12','43.255.228.13','43.255.228.14','43.255.228.15','43.255.228.16',\
'43.255.228.17') \
then '佛山互联BGP_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('43.255.228.18','43.255.228.19','43.255.228.20') \
then '佛山互联BGP_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('43.255.228.21') \
then '佛山互联BGP_GSLB' else '其它' end as access_type from dw_log_app_api_monitor_ds
Thanks
Zhangminglei
... [show rest of quote]
zhangminglei
Reply | Threaded
Open this post in threaded view
♦
♦
| More
Print post
Permalink
Jun 19, 2018; 2:46pm

Re: DataStreamCalcRule$1802" grows beyond 64 KB when execute long sql.

zhangminglei
50 posts
Hi, Fabian, Absolutely, Flink 1.5.0 I am using for this. A big CASE WHEN statement. Is it hard to implement ? I am a new to flink table api & sql.

Best Minglei.

在 2018年6月19日，下午10:36，Fabian Hueske <[hidden email]> 写道：

Hi,

Which version are you using? We fixed a similar issue for Flink 1.5.0.
If you can't upgrade yet, you can also implement a user-defined function that evaluates the big CASE WHEN statement.

Best, Fabian

2018-06-19 16:27 GMT+02:00 zhangminglei <[hidden email]>:
Hi, friends.

When I execute a long sql and get the follow error, how can I have a quick fix ?

org.apache.flink.api.common.InvalidProgramException: Table program cannot be compiled. This is a bug. Please file an issue.
        at org.apache.flink.table.codegen.Compiler$class.compile(Compiler.scala:36)
        at org.apache.flink.table.runtime.CRowProcessRunner.compile(CRowProcessRunner.scala:35)
        at org.apache.flink.table.runtime.CRowProcessRunner.open(CRowProcessRunner.scala:49)
        at org.apache.flink.api.common.functions.util.FunctionUtils.openFunction(FunctionUtils.java:36)
        at org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator.open(AbstractUdfStreamOperator.java:102)
        at org.apache.flink.streaming.api.operators.ProcessOperator.open(ProcessOperator.java:56)
        at org.apache.flink.streaming.runtime.tasks.StreamTask.openAllOperators(StreamTask.java:393)
        at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:254)
        at org.apache.flink.runtime.taskmanager.Task.run(Task.java:718)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: Compiling "DataStreamCalcRule$1802": Code of method "processElement(Ljava/lang/Object;Lorg/apache/flink/streaming/api/functions/ProcessFunction$Context;Lorg/apache/flink/util/Collector;)V" of class "DataStreamCalcRule$1802" grows beyond 64 KB
        at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:361)

select case when cast(status as bigint) in (200) then 10 else 1 end as pv,\
case when cast(status as bigint) between 303 and 1000 then 1 when cast(status as bigint) between 100 and 199 then 1 else 0 end as service_fail,\
case when cast(status as bigint) in (200) then 10 when cast(status as bigint) in (301,302) then 1 when cast(status as bigint) between 201 and 299 then 1 else 0 end as service_success,\
case when cast(status as bigint) not between 100 and 1000 then 1 else 0 end as network_fail,\
qqiplib(ip, 'isp') as isp,\
case when response_time - request_time <= 60000 then response_time - request_time else 0 end as response_t,\
case when response_time - request_time <= 60000 then 1 else 0 end as count_in,\
case when host in ('116.31.114.22','116.31.114.23','183.60.219.231','183.60.219.232','183.60.219.235','183.60.219.236','183.60.220.231','183.60.220.232',\
'183.60.219.247','183.60.219.248','183.60.219.243','183.60.219.244','183.60.219.251','183.60.219.252','116.31.114.202','116.31.114.204','116.31.114.206'\
,'116.31.114.208') then '佛山力通电信_SR' \
when host in ('183.232.169.11','183.232.169.12','183.232.169.13','183.232.169.14','183.232.169.15','183.232.169.16','183.232.169.17','183.232.169.18') \
then '佛山力通移动_SR' \
when host in ('112.93.112.11','112.93.112.12','112.93.112.13','112.93.112.14','112.93.112.15','112.93.112.16','112.93.112.17','112.93.112.18') \
then '佛山力通联通_SR' \
when host in ('114.67.56.79','114.67.56.80','114.67.56.83','114.67.56.84','114.67.56.87','114.67.56.88','114.67.56.112','114.67.56.113','114.67.56.116',\
'114.67.56.117','114.67.60.214','114.67.60.215','114.67.54.111','114.67.54.112','114.67.56.95','114.67.56.96','114.67.54.12','114.67.54.13',\
'114.67.56.93') \
then '佛山力通BGP_SR' \
when host in ('114.67.56.94','114.67.56.102','114.67.56.103','114.67.56.106','114.67.56.107','183.60.220.231','183.60.220.232','183.60.219.247',\
'183.60.219.248','114.67.60.201','114.67.60.203','114.67.60.205','114.67.60.207') \
then '佛山力通BGP_SR' \
when host in ('183.240.167.24','183.240.167.25','183.240.167.26','183.240.167.27','183.240.167.28','183.240.167.29','183.240.167.30','183.240.167.31') \
then '佛山互联移动_SR' when host in ('27.45.160.24','27.45.160.25','27.45.160.26','27.45.160.27','27.45.160.28','27.45.160.29','27.45.160.30','27.45.160.31') \
then '佛山互联联通_SR' \
when host in ('43.255.228.11','43.255.228.12','43.255.228.13','43.255.228.14','43.255.228.15','43.255.228.16','43.255.228.17','43.255.228.18','43.255.228.19',\
'43.255.228.20','43.255.228.21','43.255.228.22','43.255.228.23') \
then '佛山互联BGP_SR' \
when host in ('43.255.228.24','43.255.228.25','43.255.228.26','43.255.228.27','43.255.228.28','43.255.228.29','43.255.228.30','43.255.228.31','43.255.228.32',\
'43.255.228.33','43.255.228.34') \
then '佛山互联BGP_SR' \
when host in ('14.17.91.113','14.17.91.114','14.17.72.11','14.17.72.12','14.17.72.15','14.17.72.16','14.17.72.19','14.17.72.20','183.61.170.121','183.61.170.122',\
'14.17.91.120','14.17.91.121','14.17.72.79','14.17.72.80','14.17.72.81','14.17.72.82') \
then '东莞电信_SR' \
when host in ('221.228.213.94','221.228.213.95','221.228.213.68','221.228.213.69','221.228.213.76','221.228.213.77','221.228.213.98','221.228.213.99',\
'221.228.213.122','221.228.213.124') \
then '无锡电信_SR' \
when host in ('43.247.88.49','43.247.88.50','43.247.88.135','43.247.88.136','43.247.88.16','43.247.88.17','43.247.88.143','43.247.88.144','43.247.88.183','43.247.88.185',\
'43.247.88.187','43.247.88.189') \
then '无锡BGP_SR' \
when host in ('101.251.216.140','101.251.216.141','101.251.216.157','101.251.216.158') \
then '北京BGP_SR' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('116.31.114.22','116.31.114.23','183.60.219.231','183.60.219.232','183.60.219.235','183.60.219.236','183.60.220.231',\
'183.60.220.232','183.60.219.247','183.60.219.248','183.60.219.243','183.60.219.244','183.60.219.251','183.60.219.252','116.31.114.202','116.31.114.204',\
'116.31.114.206','116.31.114.208') \
then '佛山力通电信_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('183.232.169.11','183.232.169.12','183.232.169.13','183.232.169.14','183.232.169.15','183.232.169.16',\
'183.232.169.17','183.232.169.18') \
then '佛山力通移动_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('112.93.112.11','112.93.112.12','112.93.112.13','112.93.112.14','112.93.112.15','112.93.112.16','112.93.112.17','112.93.112.18') \
then '佛山力通联通_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('114.67.56.79','114.67.56.80','114.67.56.83','114.67.56.84','114.67.56.87','114.67.56.88','114.67.56.112',\
'114.67.56.113','114.67.56.116','114.67.56.117','114.67.60.214','114.67.60.215','114.67.54.111') \
then '佛山力通BGP_GSLB' \
when host in ('114.67.54.112','114.67.56.95','114.67.56.96','114.67.54.12','114.67.54.13','114.67.56.93','114.67.56.94','114.67.56.102','114.67.56.103','114.67.56.106',\
'114.67.56.107','183.60.220.231','183.60.220.232','183.60.219.247','183.60.219.248','114.67.60.201','114.67.60.203','114.67.60.205','114.67.60.207') \
then '佛山力通BGP_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('183.240.167.24','183.240.167.25','183.240.167.26','183.240.167.27','183.240.167.28','183.240.167.29',\
'183.240.167.30','183.240.167.31') \
then '佛山互联移动_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('43.255.228.11','43.255.228.12','43.255.228.13','43.255.228.14','43.255.228.15','43.255.228.16',\
'43.255.228.17') \
then '佛山互联BGP_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('43.255.228.18','43.255.228.19','43.255.228.20') \
then '佛山互联BGP_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('43.255.228.21') \
then '佛山互联BGP_GSLB' else '其它' end as access_type from dw_log_app_api_monitor_ds
Thanks
Zhangminglei
... [show rest of quote]

... [show rest of quote]
Fabian Hueske-2
Reply | Threaded
Open this post in threaded view
♦
♦
| More
Print post
Permalink
Jun 19, 2018; 2:52pm

Re: DataStreamCalcRule$1802" grows beyond 64 KB when execute long sql.

Fabian Hueske-2
1749 posts
I see, then this case wasn't covered by the fix that we added for Flink 1.5.0.
I guess the problem is that the code is needed to evaluate a single field.

Implementing a scalar user-function is not very difficult [1].
However, you need to register it in the TableEnvironment before you can use it in a SQL query.

Best, Fabian


2018-06-19 16:46 GMT+02:00 zhangminglei <[hidden email]>:
Hi, Fabian, Absolutely, Flink 1.5.0 I am using for this. A big CASE WHEN statement. Is it hard to implement ? I am a new to flink table api & sql.

Best Minglei.

在 2018年6月19日，下午10:36，Fabian Hueske <[hidden email]> 写道：

Hi,

Which version are you using? We fixed a similar issue for Flink 1.5.0.
If you can't upgrade yet, you can also implement a user-defined function that evaluates the big CASE WHEN statement.

Best, Fabian

2018-06-19 16:27 GMT+02:00 zhangminglei <[hidden email]>:
Hi, friends.

When I execute a long sql and get the follow error, how can I have a quick fix ?

org.apache.flink.api.common.InvalidProgramException: Table program cannot be compiled. This is a bug. Please file an issue.
        at org.apache.flink.table.codegen.Compiler$class.compile(Compiler.scala:36)
        at org.apache.flink.table.runtime.CRowProcessRunner.compile(CRowProcessRunner.scala:35)
        at org.apache.flink.table.runtime.CRowProcessRunner.open(CRowProcessRunner.scala:49)
        at org.apache.flink.api.common.functions.util.FunctionUtils.openFunction(FunctionUtils.java:36)
        at org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator.open(AbstractUdfStreamOperator.java:102)
        at org.apache.flink.streaming.api.operators.ProcessOperator.open(ProcessOperator.java:56)
        at org.apache.flink.streaming.runtime.tasks.StreamTask.openAllOperators(StreamTask.java:393)
        at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:254)
        at org.apache.flink.runtime.taskmanager.Task.run(Task.java:718)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: Compiling "DataStreamCalcRule$1802": Code of method "processElement(Ljava/lang/Object;Lorg/apache/flink/streaming/api/functions/ProcessFunction$Context;Lorg/apache/flink/util/Collector;)V" of class "DataStreamCalcRule$1802" grows beyond 64 KB
        at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:361)

select case when cast(status as bigint) in (200) then 10 else 1 end as pv,\
case when cast(status as bigint) between 303 and 1000 then 1 when cast(status as bigint) between 100 and 199 then 1 else 0 end as service_fail,\
case when cast(status as bigint) in (200) then 10 when cast(status as bigint) in (301,302) then 1 when cast(status as bigint) between 201 and 299 then 1 else 0 end as service_success,\
case when cast(status as bigint) not between 100 and 1000 then 1 else 0 end as network_fail,\
qqiplib(ip, 'isp') as isp,\
case when response_time - request_time <= 60000 then response_time - request_time else 0 end as response_t,\
case when response_time - request_time <= 60000 then 1 else 0 end as count_in,\
case when host in ('116.31.114.22','116.31.114.23','183.60.219.231','183.60.219.232','183.60.219.235','183.60.219.236','183.60.220.231','183.60.220.232',\
'183.60.219.247','183.60.219.248','183.60.219.243','183.60.219.244','183.60.219.251','183.60.219.252','116.31.114.202','116.31.114.204','116.31.114.206'\
,'116.31.114.208') then '佛山力通电信_SR' \
when host in ('183.232.169.11','183.232.169.12','183.232.169.13','183.232.169.14','183.232.169.15','183.232.169.16','183.232.169.17','183.232.169.18') \
then '佛山力通移动_SR' \
when host in ('112.93.112.11','112.93.112.12','112.93.112.13','112.93.112.14','112.93.112.15','112.93.112.16','112.93.112.17','112.93.112.18') \
then '佛山力通联通_SR' \
when host in ('114.67.56.79','114.67.56.80','114.67.56.83','114.67.56.84','114.67.56.87','114.67.56.88','114.67.56.112','114.67.56.113','114.67.56.116',\
'114.67.56.117','114.67.60.214','114.67.60.215','114.67.54.111','114.67.54.112','114.67.56.95','114.67.56.96','114.67.54.12','114.67.54.13',\
'114.67.56.93') \
then '佛山力通BGP_SR' \
when host in ('114.67.56.94','114.67.56.102','114.67.56.103','114.67.56.106','114.67.56.107','183.60.220.231','183.60.220.232','183.60.219.247',\
'183.60.219.248','114.67.60.201','114.67.60.203','114.67.60.205','114.67.60.207') \
then '佛山力通BGP_SR' \
when host in ('183.240.167.24','183.240.167.25','183.240.167.26','183.240.167.27','183.240.167.28','183.240.167.29','183.240.167.30','183.240.167.31') \
then '佛山互联移动_SR' when host in ('27.45.160.24','27.45.160.25','27.45.160.26','27.45.160.27','27.45.160.28','27.45.160.29','27.45.160.30','27.45.160.31') \
then '佛山互联联通_SR' \
when host in ('43.255.228.11','43.255.228.12','43.255.228.13','43.255.228.14','43.255.228.15','43.255.228.16','43.255.228.17','43.255.228.18','43.255.228.19',\
'43.255.228.20','43.255.228.21','43.255.228.22','43.255.228.23') \
then '佛山互联BGP_SR' \
when host in ('43.255.228.24','43.255.228.25','43.255.228.26','43.255.228.27','43.255.228.28','43.255.228.29','43.255.228.30','43.255.228.31','43.255.228.32',\
'43.255.228.33','43.255.228.34') \
then '佛山互联BGP_SR' \
when host in ('14.17.91.113','14.17.91.114','14.17.72.11','14.17.72.12','14.17.72.15','14.17.72.16','14.17.72.19','14.17.72.20','183.61.170.121','183.61.170.122',\
'14.17.91.120','14.17.91.121','14.17.72.79','14.17.72.80','14.17.72.81','14.17.72.82') \
then '东莞电信_SR' \
when host in ('221.228.213.94','221.228.213.95','221.228.213.68','221.228.213.69','221.228.213.76','221.228.213.77','221.228.213.98','221.228.213.99',\
'221.228.213.122','221.228.213.124') \
then '无锡电信_SR' \
when host in ('43.247.88.49','43.247.88.50','43.247.88.135','43.247.88.136','43.247.88.16','43.247.88.17','43.247.88.143','43.247.88.144','43.247.88.183','43.247.88.185',\
'43.247.88.187','43.247.88.189') \
then '无锡BGP_SR' \
when host in ('101.251.216.140','101.251.216.141','101.251.216.157','101.251.216.158') \
then '北京BGP_SR' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('116.31.114.22','116.31.114.23','183.60.219.231','183.60.219.232','183.60.219.235','183.60.219.236','183.60.220.231',\
'183.60.220.232','183.60.219.247','183.60.219.248','183.60.219.243','183.60.219.244','183.60.219.251','183.60.219.252','116.31.114.202','116.31.114.204',\
'116.31.114.206','116.31.114.208') \
then '佛山力通电信_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('183.232.169.11','183.232.169.12','183.232.169.13','183.232.169.14','183.232.169.15','183.232.169.16',\
'183.232.169.17','183.232.169.18') \
then '佛山力通移动_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('112.93.112.11','112.93.112.12','112.93.112.13','112.93.112.14','112.93.112.15','112.93.112.16','112.93.112.17','112.93.112.18') \
then '佛山力通联通_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('114.67.56.79','114.67.56.80','114.67.56.83','114.67.56.84','114.67.56.87','114.67.56.88','114.67.56.112',\
'114.67.56.113','114.67.56.116','114.67.56.117','114.67.60.214','114.67.60.215','114.67.54.111') \
then '佛山力通BGP_GSLB' \
when host in ('114.67.54.112','114.67.56.95','114.67.56.96','114.67.54.12','114.67.54.13','114.67.56.93','114.67.56.94','114.67.56.102','114.67.56.103','114.67.56.106',\
'114.67.56.107','183.60.220.231','183.60.220.232','183.60.219.247','183.60.219.248','114.67.60.201','114.67.60.203','114.67.60.205','114.67.60.207') \
then '佛山力通BGP_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('183.240.167.24','183.240.167.25','183.240.167.26','183.240.167.27','183.240.167.28','183.240.167.29',\
'183.240.167.30','183.240.167.31') \
then '佛山互联移动_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('43.255.228.11','43.255.228.12','43.255.228.13','43.255.228.14','43.255.228.15','43.255.228.16',\
'43.255.228.17') \
then '佛山互联BGP_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('43.255.228.18','43.255.228.19','43.255.228.20') \
then '佛山互联BGP_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('43.255.228.21') \
then '佛山互联BGP_GSLB' else '其它' end as access_type from dw_log_app_api_monitor_ds
Thanks
Zhangminglei
... [show rest of quote]

... [show rest of quote]

... [show rest of quote]
Timo Walther
Reply | Threaded
Open this post in threaded view
♦
♦
| More
Print post
Permalink
Jun 19, 2018; 4:10pm

Re: DataStreamCalcRule$1802" grows beyond 64 KB when execute long sql.

Timo Walther
644 posts
In reply to this post by zhangminglei
Hi,

this is a known issue that is mentioned in https://issues.apache.org/jira/browse/FLINK-8921 and should be fixed soon. Currently, we only split by field but for your case we should also split expressions. As a workaround you could implement your own scalar UDF that contains the case/when logic.

Regards,
Timo


Am 19.06.18 um 07:27 schrieb zhangminglei:
Hi, friends.

When I execute a long sql and get the follow error, how can I have a quick fix ?

org.apache.flink.api.common.InvalidProgramException: Table program cannot be compiled. This is a bug. Please file an issue.
at org.apache.flink.table.codegen.Compiler$class.compile(Compiler.scala:36)
at org.apache.flink.table.runtime.CRowProcessRunner.compile(CRowProcessRunner.scala:35)
at org.apache.flink.table.runtime.CRowProcessRunner.open(CRowProcessRunner.scala:49)
at org.apache.flink.api.common.functions.util.FunctionUtils.openFunction(FunctionUtils.java:36)
at org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator.open(AbstractUdfStreamOperator.java:102)
at org.apache.flink.streaming.api.operators.ProcessOperator.open(ProcessOperator.java:56)
at org.apache.flink.streaming.runtime.tasks.StreamTask.openAllOperators(StreamTask.java:393)
at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:254)
at org.apache.flink.runtime.taskmanager.Task.run(Task.java:718)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: Compiling "DataStreamCalcRule$1802": Code of method "processElement(Ljava/lang/Object;Lorg/apache/flink/streaming/api/functions/ProcessFunction$Context;Lorg/apache/flink/util/Collector;)V" of class "DataStreamCalcRule$1802" grows beyond 64 KB
at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:361)

select case when cast(status as bigint) in (200) then 10 else 1 end as pv,\
case when cast(status as bigint) between 303 and 1000 then 1 when cast(status as bigint) between 100 and 199 then 1 else 0 end as service_fail,\
case when cast(status as bigint) in (200) then 10 when cast(status as bigint) in (301,302) then 1 when cast(status as bigint) between 201 and 299 then 1 else 0 end as service_success,\
case when cast(status as bigint) not between 100 and 1000 then 1 else 0 end as network_fail,\
qqiplib(ip, 'isp') as isp,\
case when response_time - request_time <= 60000 then response_time - request_time else 0 end as response_t,\
case when response_time - request_time <= 60000 then 1 else 0 end as count_in,\
case when host in ('116.31.114.22','116.31.114.23','183.60.219.231','183.60.219.232','183.60.219.235','183.60.219.236','183.60.220.231','183.60.220.232',\
'183.60.219.247','183.60.219.248','183.60.219.243','183.60.219.244','183.60.219.251','183.60.219.252','116.31.114.202','116.31.114.204','116.31.114.206'\
,'116.31.114.208') then '佛山力通电信_SR' \
when host in ('183.232.169.11','183.232.169.12','183.232.169.13','183.232.169.14','183.232.169.15','183.232.169.16','183.232.169.17','183.232.169.18') \
then '佛山力通移动_SR' \
when host in ('112.93.112.11','112.93.112.12','112.93.112.13','112.93.112.14','112.93.112.15','112.93.112.16','112.93.112.17','112.93.112.18') \
then '佛山力通联通_SR' \
when host in ('114.67.56.79','114.67.56.80','114.67.56.83','114.67.56.84','114.67.56.87','114.67.56.88','114.67.56.112','114.67.56.113','114.67.56.116',\
'114.67.56.117','114.67.60.214','114.67.60.215','114.67.54.111','114.67.54.112','114.67.56.95','114.67.56.96','114.67.54.12','114.67.54.13',\
'114.67.56.93') \
then '佛山力通BGP_SR' \
when host in ('114.67.56.94','114.67.56.102','114.67.56.103','114.67.56.106','114.67.56.107','183.60.220.231','183.60.220.232','183.60.219.247',\
'183.60.219.248','114.67.60.201','114.67.60.203','114.67.60.205','114.67.60.207') \
then '佛山力通BGP_SR' \
when host in ('183.240.167.24','183.240.167.25','183.240.167.26','183.240.167.27','183.240.167.28','183.240.167.29','183.240.167.30','183.240.167.31') \
then '佛山互联移动_SR' when host in ('27.45.160.24','27.45.160.25','27.45.160.26','27.45.160.27','27.45.160.28','27.45.160.29','27.45.160.30','27.45.160.31') \
then '佛山互联联通_SR' \
when host in ('43.255.228.11','43.255.228.12','43.255.228.13','43.255.228.14','43.255.228.15','43.255.228.16','43.255.228.17','43.255.228.18','43.255.228.19',\
'43.255.228.20','43.255.228.21','43.255.228.22','43.255.228.23') \
then '佛山互联BGP_SR' \
when host in ('43.255.228.24','43.255.228.25','43.255.228.26','43.255.228.27','43.255.228.28','43.255.228.29','43.255.228.30','43.255.228.31','43.255.228.32',\
'43.255.228.33','43.255.228.34') \
then '佛山互联BGP_SR' \
when host in ('14.17.91.113','14.17.91.114','14.17.72.11','14.17.72.12','14.17.72.15','14.17.72.16','14.17.72.19','14.17.72.20','183.61.170.121','183.61.170.122',\
'14.17.91.120','14.17.91.121','14.17.72.79','14.17.72.80','14.17.72.81','14.17.72.82') \
then '东莞电信_SR' \
when host in ('221.228.213.94','221.228.213.95','221.228.213.68','221.228.213.69','221.228.213.76','221.228.213.77','221.228.213.98','221.228.213.99',\
'221.228.213.122','221.228.213.124') \
then '无锡电信_SR' \
when host in ('43.247.88.49','43.247.88.50','43.247.88.135','43.247.88.136','43.247.88.16','43.247.88.17','43.247.88.143','43.247.88.144','43.247.88.183','43.247.88.185',\
'43.247.88.187','43.247.88.189') \
then '无锡BGP_SR' \
when host in ('101.251.216.140','101.251.216.141','101.251.216.157','101.251.216.158') \
then '北京BGP_SR' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('116.31.114.22','116.31.114.23','183.60.219.231','183.60.219.232','183.60.219.235','183.60.219.236','183.60.220.231',\
'183.60.220.232','183.60.219.247','183.60.219.248','183.60.219.243','183.60.219.244','183.60.219.251','183.60.219.252','116.31.114.202','116.31.114.204',\
'116.31.114.206','116.31.114.208') \
then '佛山力通电信_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('183.232.169.11','183.232.169.12','183.232.169.13','183.232.169.14','183.232.169.15','183.232.169.16',\
'183.232.169.17','183.232.169.18') \
then '佛山力通移动_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('112.93.112.11','112.93.112.12','112.93.112.13','112.93.112.14','112.93.112.15','112.93.112.16','112.93.112.17','112.93.112.18') \
then '佛山力通联通_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('114.67.56.79','114.67.56.80','114.67.56.83','114.67.56.84','114.67.56.87','114.67.56.88','114.67.56.112',\
'114.67.56.113','114.67.56.116','114.67.56.117','114.67.60.214','114.67.60.215','114.67.54.111') \
then '佛山力通BGP_GSLB' \
when host in ('114.67.54.112','114.67.56.95','114.67.56.96','114.67.54.12','114.67.54.13','114.67.56.93','114.67.56.94','114.67.56.102','114.67.56.103','114.67.56.106',\
'114.67.56.107','183.60.220.231','183.60.220.232','183.60.219.247','183.60.219.248','114.67.60.201','114.67.60.203','114.67.60.205','114.67.60.207') \
then '佛山力通BGP_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('183.240.167.24','183.240.167.25','183.240.167.26','183.240.167.27','183.240.167.28','183.240.167.29',\
'183.240.167.30','183.240.167.31') \
then '佛山互联移动_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('43.255.228.11','43.255.228.12','43.255.228.13','43.255.228.14','43.255.228.15','43.255.228.16',\
'43.255.228.17') \
then '佛山互联BGP_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('43.255.228.18','43.255.228.19','43.255.228.20') \
then '佛山互联BGP_GSLB' \
when host in ('mapi.appvipshop.com') and mapi_ip in ('43.255.228.21') \
then '佛山互联BGP_GSLB' else '其它' end as access_type from dw_log_app_api_monitor_ds
Thanks
Zhangminglei
... [show rest of quote]
Free forum by Nabble Edit this page