dbhose-airflow 0.0.2.1__py3-none-any.whl → 0.0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbhose_airflow/LOGO +9 -0
- dbhose_airflow/__init__.py +1 -1
- dbhose_airflow/ddl/clickhouse.sql +81 -0
- dbhose_airflow/ddl/greenplum.sql +229 -0
- dbhose_airflow/ddl/postgres.sql +209 -0
- dbhose_airflow/dq/clickhouse/empty.sql +1 -0
- dbhose_airflow/dq/clickhouse/future.sql +34 -0
- dbhose_airflow/dq/clickhouse/infinity.sql +34 -0
- dbhose_airflow/dq/clickhouse/nan.sql +34 -0
- dbhose_airflow/dq/clickhouse/sum.sql +72 -0
- dbhose_airflow/dq/clickhouse/total.sql +1 -0
- dbhose_airflow/dq/clickhouse/uniq.sql +1 -0
- dbhose_airflow/dq/greenplum/empty.sql +1 -0
- dbhose_airflow/dq/greenplum/future.sql +55 -0
- dbhose_airflow/dq/greenplum/infinity.sql +60 -0
- dbhose_airflow/dq/greenplum/nan.sql +58 -0
- dbhose_airflow/dq/greenplum/sum.sql +63 -0
- dbhose_airflow/dq/greenplum/total.sql +1 -0
- dbhose_airflow/dq/greenplum/uniq.sql +2 -0
- dbhose_airflow/dq/postgres/empty.sql +1 -0
- dbhose_airflow/dq/postgres/future.sql +55 -0
- dbhose_airflow/dq/postgres/infinity.sql +60 -0
- dbhose_airflow/dq/postgres/nan.sql +58 -0
- dbhose_airflow/dq/postgres/sum.sql +63 -0
- dbhose_airflow/dq/postgres/total.sql +1 -0
- dbhose_airflow/dq/postgres/uniq.sql +2 -0
- dbhose_airflow/move/clickhouse/delete.sql +81 -0
- dbhose_airflow/move/clickhouse/replace.sql +24 -0
- dbhose_airflow/move/greenplum/delete.sql +10 -0
- dbhose_airflow/move/greenplum/replace.sql +60 -0
- dbhose_airflow/move/postgres/delete.sql +10 -0
- dbhose_airflow/move/postgres/replace.sql +60 -0
- {dbhose_airflow-0.0.2.1.dist-info → dbhose_airflow-0.0.2.2.dist-info}/METADATA +1 -1
- dbhose_airflow-0.0.2.2.dist-info/RECORD +43 -0
- {dbhose_airflow-0.0.2.1.dist-info → dbhose_airflow-0.0.2.2.dist-info}/licenses/CHANGELOG.md +4 -0
- dbhose_airflow-0.0.2.1.dist-info/RECORD +0 -12
- {dbhose_airflow-0.0.2.1.dist-info → dbhose_airflow-0.0.2.2.dist-info}/WHEEL +0 -0
- {dbhose_airflow-0.0.2.1.dist-info → dbhose_airflow-0.0.2.2.dist-info}/licenses/README.md +0 -0
- {dbhose_airflow-0.0.2.1.dist-info → dbhose_airflow-0.0.2.2.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
with
|
|
2
|
+
replaceAll('{table}', '`', '') as table_name
|
|
3
|
+
, splitByChar('.', table_name) as table_parts
|
|
4
|
+
select
|
|
5
|
+
have_test
|
|
6
|
+
, column_name
|
|
7
|
+
, query
|
|
8
|
+
from (
|
|
9
|
+
select
|
|
10
|
+
database
|
|
11
|
+
, table
|
|
12
|
+
, toBool(countIf(type in (
|
|
13
|
+
'Int8'
|
|
14
|
+
, 'Int16'
|
|
15
|
+
, 'Int32'
|
|
16
|
+
, 'Int64'
|
|
17
|
+
, 'Int128'
|
|
18
|
+
, 'Int256'
|
|
19
|
+
, 'UInt8'
|
|
20
|
+
, 'UInt16'
|
|
21
|
+
, 'UInt32'
|
|
22
|
+
, 'UInt64'
|
|
23
|
+
, 'UInt128'
|
|
24
|
+
, 'UInt256'
|
|
25
|
+
, 'Float32'
|
|
26
|
+
, 'Float64'
|
|
27
|
+
, 'Decimal32'
|
|
28
|
+
, 'Decimal64'
|
|
29
|
+
, 'Decimal128'
|
|
30
|
+
, 'Decimal256'
|
|
31
|
+
))) as have_test
|
|
32
|
+
from system.columns
|
|
33
|
+
where database = table_parts[1]
|
|
34
|
+
and table = table_parts[2]
|
|
35
|
+
group by 1, 2
|
|
36
|
+
) as ht
|
|
37
|
+
left join (
|
|
38
|
+
select
|
|
39
|
+
name as column_name
|
|
40
|
+
, format(
|
|
41
|
+
'select round(sum({{}}), 2) as value \n' ||
|
|
42
|
+
'from {{}}'
|
|
43
|
+
, column_name
|
|
44
|
+
, table_name
|
|
45
|
+
) as query
|
|
46
|
+
, database
|
|
47
|
+
, table
|
|
48
|
+
from system.columns
|
|
49
|
+
where database = table_parts[1]
|
|
50
|
+
and table = table_parts[2]
|
|
51
|
+
and type in (
|
|
52
|
+
'Int8'
|
|
53
|
+
, 'Int16'
|
|
54
|
+
, 'Int32'
|
|
55
|
+
, 'Int64'
|
|
56
|
+
, 'Int128'
|
|
57
|
+
, 'Int256'
|
|
58
|
+
, 'UInt8'
|
|
59
|
+
, 'UInt16'
|
|
60
|
+
, 'UInt32'
|
|
61
|
+
, 'UInt64'
|
|
62
|
+
, 'UInt128'
|
|
63
|
+
, 'UInt256'
|
|
64
|
+
, 'Float32'
|
|
65
|
+
, 'Float64'
|
|
66
|
+
, 'Decimal32'
|
|
67
|
+
, 'Decimal64'
|
|
68
|
+
, 'Decimal128'
|
|
69
|
+
, 'Decimal256'
|
|
70
|
+
)
|
|
71
|
+
) as qt
|
|
72
|
+
using(database, table)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
select count(*) as value from {table}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
select count(*) - countDistinct(*) as value, case when value = 0 then 'Pass' else 'Fail' end as result from {table}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
select count(*) as value, case when count(*) = 0 then 'Fail' else 'Pass' end as result from {table}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
with cte as (
|
|
2
|
+
select '{table}'::regclass as table_class
|
|
3
|
+
)
|
|
4
|
+
select
|
|
5
|
+
have_test
|
|
6
|
+
, column_name
|
|
7
|
+
, query
|
|
8
|
+
from (
|
|
9
|
+
select
|
|
10
|
+
n.nspname::text as schema_name
|
|
11
|
+
, c.relname::text as table_name
|
|
12
|
+
, count(*) filter(
|
|
13
|
+
where format_type(a.atttypid, a.atttypmod) in (
|
|
14
|
+
'date'
|
|
15
|
+
, 'timestamp with time zone'
|
|
16
|
+
, 'timestamp without time zone'
|
|
17
|
+
)
|
|
18
|
+
) > 0 as have_test
|
|
19
|
+
from pg_class as c
|
|
20
|
+
join pg_namespace as n
|
|
21
|
+
on n.oid = c.relnamespace
|
|
22
|
+
join pg_attribute as a
|
|
23
|
+
on a.attrelid = c.oid
|
|
24
|
+
where c.oid = (select table_class from cte)
|
|
25
|
+
and a.attnum > 0
|
|
26
|
+
and not a.attisdropped
|
|
27
|
+
group by 1, 2
|
|
28
|
+
) as ht
|
|
29
|
+
left join (
|
|
30
|
+
select
|
|
31
|
+
a.attname::text as column_name
|
|
32
|
+
, n.nspname::text as schema_name
|
|
33
|
+
, c.relname::text as table_name
|
|
34
|
+
, format(
|
|
35
|
+
e'select count(*) as value, case when count(*) = 0 then ''Pass'' else ''Fail'' end as result \n' ||
|
|
36
|
+
e'from %I.%I where %I > (current_date + interval ''1 month'')'
|
|
37
|
+
, n.nspname
|
|
38
|
+
, c.relname
|
|
39
|
+
, a.attname
|
|
40
|
+
) as query
|
|
41
|
+
from pg_attribute as a
|
|
42
|
+
join pg_class as c
|
|
43
|
+
on a.attrelid = c.oid
|
|
44
|
+
join pg_namespace as n
|
|
45
|
+
on n.oid = c.relnamespace
|
|
46
|
+
where c.oid = (select table_class from cte)
|
|
47
|
+
and a.attnum > 0
|
|
48
|
+
and not a.attisdropped
|
|
49
|
+
and format_type(a.atttypid, a.atttypmod) in (
|
|
50
|
+
'date'
|
|
51
|
+
, 'timestamp with time zone'
|
|
52
|
+
, 'timestamp without time zone'
|
|
53
|
+
)
|
|
54
|
+
) as qt
|
|
55
|
+
using(schema_name, table_name)
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
with cte as (
|
|
2
|
+
select
|
|
3
|
+
'{table}'::regclass as table_class
|
|
4
|
+
)
|
|
5
|
+
select
|
|
6
|
+
have_test
|
|
7
|
+
, column_name
|
|
8
|
+
, query
|
|
9
|
+
from (
|
|
10
|
+
select
|
|
11
|
+
n.nspname::text as schema_name
|
|
12
|
+
, c.relname::text as table_name
|
|
13
|
+
, count(*) filter(
|
|
14
|
+
where format_type(a.atttypid, a.atttypmod) in (
|
|
15
|
+
'date'
|
|
16
|
+
, 'double precision'
|
|
17
|
+
, 'real'
|
|
18
|
+
, 'timestamp with time zone'
|
|
19
|
+
, 'timestamp without time zone'
|
|
20
|
+
)
|
|
21
|
+
) > 0 as have_test
|
|
22
|
+
from pg_class as c
|
|
23
|
+
join pg_namespace as n
|
|
24
|
+
on n.oid = c.relnamespace
|
|
25
|
+
join pg_attribute as a
|
|
26
|
+
on a.attrelid = c.oid
|
|
27
|
+
where c.oid = (select table_class from cte)
|
|
28
|
+
and a.attnum > 0
|
|
29
|
+
and not a.attisdropped
|
|
30
|
+
group by 1, 2
|
|
31
|
+
) as ht
|
|
32
|
+
left join (
|
|
33
|
+
select
|
|
34
|
+
a.attname::text as column_name
|
|
35
|
+
, n.nspname::text as schema_name
|
|
36
|
+
, c.relname::text as table_name
|
|
37
|
+
, format(
|
|
38
|
+
E'select count(*) as value, case when count(*) = 0 then ''Pass'' else ''Fail'' end as result \n' ||
|
|
39
|
+
E'from %I.%I where %I in (''infinity'', ''-infinity'')'
|
|
40
|
+
, n.nspname
|
|
41
|
+
, c.relname
|
|
42
|
+
, a.attname
|
|
43
|
+
) as query
|
|
44
|
+
from pg_attribute as a
|
|
45
|
+
join pg_class as c
|
|
46
|
+
on a.attrelid = c.oid
|
|
47
|
+
join pg_namespace as n
|
|
48
|
+
on n.oid = c.relnamespace
|
|
49
|
+
where c.oid = (select table_class from cte)
|
|
50
|
+
and a.attnum > 0
|
|
51
|
+
and not a.attisdropped
|
|
52
|
+
and format_type(a.atttypid, a.atttypmod) in (
|
|
53
|
+
'date'
|
|
54
|
+
, 'double precision'
|
|
55
|
+
, 'real'
|
|
56
|
+
, 'timestamp with time zone'
|
|
57
|
+
, 'timestamp without time zone'
|
|
58
|
+
)
|
|
59
|
+
) as qt
|
|
60
|
+
using(schema_name, table_name)
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
with cte as (
|
|
2
|
+
select
|
|
3
|
+
'{table}'::regclass as table_class
|
|
4
|
+
)
|
|
5
|
+
select
|
|
6
|
+
have_test
|
|
7
|
+
, column_name
|
|
8
|
+
, query
|
|
9
|
+
from (
|
|
10
|
+
select
|
|
11
|
+
n.nspname::text as schema_name
|
|
12
|
+
, c.relname::text as table_name
|
|
13
|
+
, count(*) filter(
|
|
14
|
+
where format_type(a.atttypid, a.atttypmod) in (
|
|
15
|
+
'decimal'
|
|
16
|
+
, 'double precision'
|
|
17
|
+
, 'real'
|
|
18
|
+
, 'numeric'
|
|
19
|
+
)
|
|
20
|
+
) > 0 as have_test
|
|
21
|
+
from pg_class as c
|
|
22
|
+
join pg_namespace as n
|
|
23
|
+
on n.oid = c.relnamespace
|
|
24
|
+
join pg_attribute as a
|
|
25
|
+
on a.attrelid = c.oid
|
|
26
|
+
where c.oid = (select table_class from cte)
|
|
27
|
+
and a.attnum > 0
|
|
28
|
+
and not a.attisdropped
|
|
29
|
+
group by 1, 2
|
|
30
|
+
) as ht
|
|
31
|
+
left join (
|
|
32
|
+
select
|
|
33
|
+
a.attname::text as column_name
|
|
34
|
+
, n.nspname::text as schema_name
|
|
35
|
+
, c.relname::text as table_name
|
|
36
|
+
, format(
|
|
37
|
+
E'select count(*) as value, case when count(*) = 0 then ''Pass'' else ''Fail'' end as result \n' ||
|
|
38
|
+
E'from %I.%I where %I = ''nan'''
|
|
39
|
+
, n.nspname
|
|
40
|
+
, c.relname
|
|
41
|
+
, a.attname
|
|
42
|
+
) as query
|
|
43
|
+
from pg_attribute as a
|
|
44
|
+
join pg_class as c
|
|
45
|
+
on a.attrelid = c.oid
|
|
46
|
+
join pg_namespace as n
|
|
47
|
+
on n.oid = c.relnamespace
|
|
48
|
+
where c.oid = (select table_class from cte)
|
|
49
|
+
and a.attnum > 0
|
|
50
|
+
and not a.attisdropped
|
|
51
|
+
and format_type(a.atttypid, a.atttypmod) in (
|
|
52
|
+
'decimal'
|
|
53
|
+
, 'double precision'
|
|
54
|
+
, 'real'
|
|
55
|
+
, 'numeric'
|
|
56
|
+
)
|
|
57
|
+
) as qt
|
|
58
|
+
using(schema_name, table_name)
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
with cte as (
|
|
2
|
+
select '{table}'::regclass as table_class
|
|
3
|
+
)
|
|
4
|
+
select
|
|
5
|
+
have_test
|
|
6
|
+
, column_name
|
|
7
|
+
, query
|
|
8
|
+
from (
|
|
9
|
+
select
|
|
10
|
+
n.nspname::text as schema_name
|
|
11
|
+
, c.relname::text as table_name
|
|
12
|
+
, count(*) filter(
|
|
13
|
+
where format_type(a.atttypid, a.atttypmod) in (
|
|
14
|
+
'smallint'
|
|
15
|
+
, 'integer'
|
|
16
|
+
, 'bigint'
|
|
17
|
+
, 'decimal'
|
|
18
|
+
, 'numeric'
|
|
19
|
+
, 'real'
|
|
20
|
+
, 'double precision'
|
|
21
|
+
)
|
|
22
|
+
) > 0 as have_test
|
|
23
|
+
from pg_class as c
|
|
24
|
+
join pg_namespace as n
|
|
25
|
+
on n.oid = c.relnamespace
|
|
26
|
+
join pg_attribute as a
|
|
27
|
+
on a.attrelid = c.oid
|
|
28
|
+
where c.oid = (select table_class from cte)
|
|
29
|
+
and a.attnum > 0
|
|
30
|
+
and not a.attisdropped
|
|
31
|
+
group by 1, 2
|
|
32
|
+
) as ht
|
|
33
|
+
left join (
|
|
34
|
+
select
|
|
35
|
+
a.attname::text as column_name
|
|
36
|
+
, n.nspname::text as schema_name
|
|
37
|
+
, c.relname::text as table_name
|
|
38
|
+
, format(
|
|
39
|
+
E'select round(sum(coalesce(%I::numeric, 0)), 2) as value \n' ||
|
|
40
|
+
E'from %I.%I'
|
|
41
|
+
, a.attname
|
|
42
|
+
, n.nspname
|
|
43
|
+
, c.relname
|
|
44
|
+
) as query
|
|
45
|
+
from pg_attribute as a
|
|
46
|
+
join pg_class as c
|
|
47
|
+
on a.attrelid = c.oid
|
|
48
|
+
join pg_namespace as n
|
|
49
|
+
on n.oid = c.relnamespace
|
|
50
|
+
where c.oid = (select table_class from cte)
|
|
51
|
+
and a.attnum > 0
|
|
52
|
+
and not a.attisdropped
|
|
53
|
+
and format_type(a.atttypid, a.atttypmod) in (
|
|
54
|
+
'smallint'
|
|
55
|
+
, 'integer'
|
|
56
|
+
, 'bigint'
|
|
57
|
+
, 'decimal'
|
|
58
|
+
, 'numeric'
|
|
59
|
+
, 'real'
|
|
60
|
+
, 'double precision'
|
|
61
|
+
)
|
|
62
|
+
) as qt
|
|
63
|
+
using(schema_name, table_name)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
select count(*) as value from {table}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
select count(*) as value, case when count(*) = 0 then 'Fail' else 'Pass' end as result from {table}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
with cte as (
|
|
2
|
+
select '{table}'::regclass as table_class
|
|
3
|
+
)
|
|
4
|
+
select
|
|
5
|
+
have_test
|
|
6
|
+
, column_name
|
|
7
|
+
, query
|
|
8
|
+
from (
|
|
9
|
+
select
|
|
10
|
+
n.nspname::text as schema_name
|
|
11
|
+
, c.relname::text as table_name
|
|
12
|
+
, count(*) filter(
|
|
13
|
+
where format_type(a.atttypid, a.atttypmod) in (
|
|
14
|
+
'date'
|
|
15
|
+
, 'timestamp with time zone'
|
|
16
|
+
, 'timestamp without time zone'
|
|
17
|
+
)
|
|
18
|
+
) > 0 as have_test
|
|
19
|
+
from pg_class as c
|
|
20
|
+
join pg_namespace as n
|
|
21
|
+
on n.oid = c.relnamespace
|
|
22
|
+
join pg_attribute as a
|
|
23
|
+
on a.attrelid = c.oid
|
|
24
|
+
where c.oid = (select table_class from cte)
|
|
25
|
+
and a.attnum > 0
|
|
26
|
+
and not a.attisdropped
|
|
27
|
+
group by 1, 2
|
|
28
|
+
) as ht
|
|
29
|
+
left join (
|
|
30
|
+
select
|
|
31
|
+
a.attname::text as column_name
|
|
32
|
+
, n.nspname::text as schema_name
|
|
33
|
+
, c.relname::text as table_name
|
|
34
|
+
, format(
|
|
35
|
+
e'select count(*) as value, case when count(*) = 0 then ''Pass'' else ''Fail'' end as result \n' ||
|
|
36
|
+
e'from %I.%I where %I > (current_date + interval ''1 month'')'
|
|
37
|
+
, n.nspname
|
|
38
|
+
, c.relname
|
|
39
|
+
, a.attname
|
|
40
|
+
) as query
|
|
41
|
+
from pg_attribute as a
|
|
42
|
+
join pg_class as c
|
|
43
|
+
on a.attrelid = c.oid
|
|
44
|
+
join pg_namespace as n
|
|
45
|
+
on n.oid = c.relnamespace
|
|
46
|
+
where c.oid = (select table_class from cte)
|
|
47
|
+
and a.attnum > 0
|
|
48
|
+
and not a.attisdropped
|
|
49
|
+
and format_type(a.atttypid, a.atttypmod) in (
|
|
50
|
+
'date'
|
|
51
|
+
, 'timestamp with time zone'
|
|
52
|
+
, 'timestamp without time zone'
|
|
53
|
+
)
|
|
54
|
+
) as qt
|
|
55
|
+
using(schema_name, table_name)
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
with cte as (
|
|
2
|
+
select
|
|
3
|
+
'{table}'::regclass as table_class
|
|
4
|
+
)
|
|
5
|
+
select
|
|
6
|
+
have_test
|
|
7
|
+
, column_name
|
|
8
|
+
, query
|
|
9
|
+
from (
|
|
10
|
+
select
|
|
11
|
+
n.nspname::text as schema_name
|
|
12
|
+
, c.relname::text as table_name
|
|
13
|
+
, count(*) filter(
|
|
14
|
+
where format_type(a.atttypid, a.atttypmod) in (
|
|
15
|
+
'date'
|
|
16
|
+
, 'double precision'
|
|
17
|
+
, 'real'
|
|
18
|
+
, 'timestamp with time zone'
|
|
19
|
+
, 'timestamp without time zone'
|
|
20
|
+
)
|
|
21
|
+
) > 0 as have_test
|
|
22
|
+
from pg_class as c
|
|
23
|
+
join pg_namespace as n
|
|
24
|
+
on n.oid = c.relnamespace
|
|
25
|
+
join pg_attribute as a
|
|
26
|
+
on a.attrelid = c.oid
|
|
27
|
+
where c.oid = (select table_class from cte)
|
|
28
|
+
and a.attnum > 0
|
|
29
|
+
and not a.attisdropped
|
|
30
|
+
group by 1, 2
|
|
31
|
+
) as ht
|
|
32
|
+
left join (
|
|
33
|
+
select
|
|
34
|
+
a.attname::text as column_name
|
|
35
|
+
, n.nspname::text as schema_name
|
|
36
|
+
, c.relname::text as table_name
|
|
37
|
+
, format(
|
|
38
|
+
E'select count(*) as value, case when count(*) = 0 then ''Pass'' else ''Fail'' end as result \n' ||
|
|
39
|
+
E'from %I.%I where %I in (''infinity'', ''-infinity'')'
|
|
40
|
+
, n.nspname
|
|
41
|
+
, c.relname
|
|
42
|
+
, a.attname
|
|
43
|
+
) as query
|
|
44
|
+
from pg_attribute as a
|
|
45
|
+
join pg_class as c
|
|
46
|
+
on a.attrelid = c.oid
|
|
47
|
+
join pg_namespace as n
|
|
48
|
+
on n.oid = c.relnamespace
|
|
49
|
+
where c.oid = (select table_class from cte)
|
|
50
|
+
and a.attnum > 0
|
|
51
|
+
and not a.attisdropped
|
|
52
|
+
and format_type(a.atttypid, a.atttypmod) in (
|
|
53
|
+
'date'
|
|
54
|
+
, 'double precision'
|
|
55
|
+
, 'real'
|
|
56
|
+
, 'timestamp with time zone'
|
|
57
|
+
, 'timestamp without time zone'
|
|
58
|
+
)
|
|
59
|
+
) as qt
|
|
60
|
+
using(schema_name, table_name)
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
with cte as (
|
|
2
|
+
select
|
|
3
|
+
'{table}'::regclass as table_class
|
|
4
|
+
)
|
|
5
|
+
select
|
|
6
|
+
have_test
|
|
7
|
+
, column_name
|
|
8
|
+
, query
|
|
9
|
+
from (
|
|
10
|
+
select
|
|
11
|
+
n.nspname::text as schema_name
|
|
12
|
+
, c.relname::text as table_name
|
|
13
|
+
, count(*) filter(
|
|
14
|
+
where format_type(a.atttypid, a.atttypmod) in (
|
|
15
|
+
'decimal'
|
|
16
|
+
, 'double precision'
|
|
17
|
+
, 'real'
|
|
18
|
+
, 'numeric'
|
|
19
|
+
)
|
|
20
|
+
) > 0 as have_test
|
|
21
|
+
from pg_class as c
|
|
22
|
+
join pg_namespace as n
|
|
23
|
+
on n.oid = c.relnamespace
|
|
24
|
+
join pg_attribute as a
|
|
25
|
+
on a.attrelid = c.oid
|
|
26
|
+
where c.oid = (select table_class from cte)
|
|
27
|
+
and a.attnum > 0
|
|
28
|
+
and not a.attisdropped
|
|
29
|
+
group by 1, 2
|
|
30
|
+
) as ht
|
|
31
|
+
left join (
|
|
32
|
+
select
|
|
33
|
+
a.attname::text as column_name
|
|
34
|
+
, n.nspname::text as schema_name
|
|
35
|
+
, c.relname::text as table_name
|
|
36
|
+
, format(
|
|
37
|
+
E'select count(*) as value, case when count(*) = 0 then ''Pass'' else ''Fail'' end as result \n' ||
|
|
38
|
+
E'from %I.%I where %I = ''nan'''
|
|
39
|
+
, n.nspname
|
|
40
|
+
, c.relname
|
|
41
|
+
, a.attname
|
|
42
|
+
) as query
|
|
43
|
+
from pg_attribute as a
|
|
44
|
+
join pg_class as c
|
|
45
|
+
on a.attrelid = c.oid
|
|
46
|
+
join pg_namespace as n
|
|
47
|
+
on n.oid = c.relnamespace
|
|
48
|
+
where c.oid = (select table_class from cte)
|
|
49
|
+
and a.attnum > 0
|
|
50
|
+
and not a.attisdropped
|
|
51
|
+
and format_type(a.atttypid, a.atttypmod) in (
|
|
52
|
+
'decimal'
|
|
53
|
+
, 'double precision'
|
|
54
|
+
, 'real'
|
|
55
|
+
, 'numeric'
|
|
56
|
+
)
|
|
57
|
+
) as qt
|
|
58
|
+
using(schema_name, table_name)
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
with cte as (
|
|
2
|
+
select '{table}'::regclass as table_class
|
|
3
|
+
)
|
|
4
|
+
select
|
|
5
|
+
have_test
|
|
6
|
+
, column_name
|
|
7
|
+
, query
|
|
8
|
+
from (
|
|
9
|
+
select
|
|
10
|
+
n.nspname::text as schema_name
|
|
11
|
+
, c.relname::text as table_name
|
|
12
|
+
, count(*) filter(
|
|
13
|
+
where format_type(a.atttypid, a.atttypmod) in (
|
|
14
|
+
'smallint'
|
|
15
|
+
, 'integer'
|
|
16
|
+
, 'bigint'
|
|
17
|
+
, 'decimal'
|
|
18
|
+
, 'numeric'
|
|
19
|
+
, 'real'
|
|
20
|
+
, 'double precision'
|
|
21
|
+
)
|
|
22
|
+
) > 0 as have_test
|
|
23
|
+
from pg_class as c
|
|
24
|
+
join pg_namespace as n
|
|
25
|
+
on n.oid = c.relnamespace
|
|
26
|
+
join pg_attribute as a
|
|
27
|
+
on a.attrelid = c.oid
|
|
28
|
+
where c.oid = (select table_class from cte)
|
|
29
|
+
and a.attnum > 0
|
|
30
|
+
and not a.attisdropped
|
|
31
|
+
group by 1, 2
|
|
32
|
+
) as ht
|
|
33
|
+
left join (
|
|
34
|
+
select
|
|
35
|
+
a.attname::text as column_name
|
|
36
|
+
, n.nspname::text as schema_name
|
|
37
|
+
, c.relname::text as table_name
|
|
38
|
+
, format(
|
|
39
|
+
E'select round(sum(coalesce(%I::numeric, 0)), 2) as value \n' ||
|
|
40
|
+
E'from %I.%I'
|
|
41
|
+
, a.attname
|
|
42
|
+
, n.nspname
|
|
43
|
+
, c.relname
|
|
44
|
+
) as query
|
|
45
|
+
from pg_attribute as a
|
|
46
|
+
join pg_class as c
|
|
47
|
+
on a.attrelid = c.oid
|
|
48
|
+
join pg_namespace as n
|
|
49
|
+
on n.oid = c.relnamespace
|
|
50
|
+
where c.oid = (select table_class from cte)
|
|
51
|
+
and a.attnum > 0
|
|
52
|
+
and not a.attisdropped
|
|
53
|
+
and format_type(a.atttypid, a.atttypmod) in (
|
|
54
|
+
'smallint'
|
|
55
|
+
, 'integer'
|
|
56
|
+
, 'bigint'
|
|
57
|
+
, 'decimal'
|
|
58
|
+
, 'numeric'
|
|
59
|
+
, 'real'
|
|
60
|
+
, 'double precision'
|
|
61
|
+
)
|
|
62
|
+
) as qt
|
|
63
|
+
using(schema_name, table_name)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
select count(*) as value from {table}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
with
|
|
2
|
+
splitByChar(',', replaceAll('{filter_by}', ' ', '')) as columns,
|
|
3
|
+
length(columns) as columns_count,
|
|
4
|
+
(
|
|
5
|
+
select groupArray(tuple({filter_by}))
|
|
6
|
+
from (
|
|
7
|
+
select distinct {filter_by}
|
|
8
|
+
from {table_temp}
|
|
9
|
+
)
|
|
10
|
+
) as values_tuples
|
|
11
|
+
select
|
|
12
|
+
true as is_avaliable,
|
|
13
|
+
'ALTER TABLE {table_dest} DELETE WHERE ' ||
|
|
14
|
+
if(columns_count > 1, '(' || arrayStringConcat(columns, ', ') || ')', arrayStringConcat(columns, ', ')) ||
|
|
15
|
+
' IN (' ||
|
|
16
|
+
arrayStringConcat(
|
|
17
|
+
arrayDistinct(
|
|
18
|
+
arrayMap(
|
|
19
|
+
tuple ->
|
|
20
|
+
if(columns_count = 1,
|
|
21
|
+
(case
|
|
22
|
+
when isNull(tuple.1) then 'NULL'
|
|
23
|
+
when toTypeName(tuple.1) REGEXP 'U?Int|B?Float|Decimal|Bool' then toString(tuple.1)
|
|
24
|
+
else '\'' || replaceAll(toString(tuple.1), '\'', '\\\'') || '\''
|
|
25
|
+
end),
|
|
26
|
+
if(columns_count = 2,
|
|
27
|
+
'(' ||
|
|
28
|
+
(case
|
|
29
|
+
when isNull(tuple.1) then 'NULL'
|
|
30
|
+
when toTypeName(tuple.1) REGEXP 'U?Int|B?Float|Decimal|Bool' then toString(tuple.1)
|
|
31
|
+
else '\'' || replaceAll(toString(tuple.1), '\'', '\\\'') || '\''
|
|
32
|
+
end) || ', ' ||
|
|
33
|
+
(case
|
|
34
|
+
when isNull(tuple.2) then 'NULL'
|
|
35
|
+
when toTypeName(tuple.2) REGEXP 'U?Int|B?Float|Decimal|Bool' then toString(tuple.2)
|
|
36
|
+
else '\'' || replaceAll(toString(tuple.2), '\'', '\\\'') || '\''
|
|
37
|
+
end) || ')',
|
|
38
|
+
if(columns_count = 3,
|
|
39
|
+
'(' ||
|
|
40
|
+
(case
|
|
41
|
+
when isNull(tuple.1) then 'NULL'
|
|
42
|
+
when toTypeName(tuple.1) REGEXP 'U?Int|B?Float|Decimal|Bool' then toString(tuple.1)
|
|
43
|
+
else '\'' || replaceAll(toString(tuple.1), '\'', '\\\'') || '\''
|
|
44
|
+
end) || ', ' ||
|
|
45
|
+
(case
|
|
46
|
+
when isNull(tuple.2) then 'NULL'
|
|
47
|
+
when toTypeName(tuple.2) REGEXP 'U?Int|B?Float|Decimal|Bool' then toString(tuple.2)
|
|
48
|
+
else '\'' || replaceAll(toString(tuple.2), '\'', '\\\'') || '\''
|
|
49
|
+
end) || ', ' ||
|
|
50
|
+
(case
|
|
51
|
+
WHEN isNull(tuple.3) THEN 'NULL'
|
|
52
|
+
WHEN toTypeName(tuple.3) REGEXP 'U?Int|B?Float|Decimal|Bool' THEN toString(tuple.3)
|
|
53
|
+
ELSE '\'' || replaceAll(toString(tuple.3), '\'', '\\\'') || '\''
|
|
54
|
+
END) || ')',
|
|
55
|
+
'(' ||
|
|
56
|
+
(CASE
|
|
57
|
+
WHEN isNull(tuple.1) THEN 'NULL'
|
|
58
|
+
WHEN toTypeName(tuple.1) REGEXP 'U?Int|B?Float|Decimal|Bool' THEN toString(tuple.1)
|
|
59
|
+
ELSE '\'' || replaceAll(toString(tuple.1), '\'', '\\\'') || '\''
|
|
60
|
+
END) || ', ' ||
|
|
61
|
+
(CASE
|
|
62
|
+
WHEN isNull(tuple.2) THEN 'NULL'
|
|
63
|
+
WHEN toTypeName(tuple.2) REGEXP 'U?Int|B?Float|Decimal|Bool' THEN toString(tuple.2)
|
|
64
|
+
ELSE '\'' || replaceAll(toString(tuple.2), '\'', '\\\'') || '\''
|
|
65
|
+
END) || ', ' ||
|
|
66
|
+
(case
|
|
67
|
+
when isNull(tuple.3) then 'NULL'
|
|
68
|
+
when toTypeName(tuple.3) REGEXP 'U?Int|B?Float|Decimal|Bool' then toString(tuple.3)
|
|
69
|
+
else '\'' || replaceAll(toString(tuple.3), '\'', '\\\'') || '\''
|
|
70
|
+
end) || ', ' ||
|
|
71
|
+
(case
|
|
72
|
+
when isNull(tuple.4) then 'NULL'
|
|
73
|
+
when toTypeName(tuple.4) REGEXP 'U?Int|B?Float|Decimal|Bool' then toString(tuple.4)
|
|
74
|
+
else '\'' || replaceAll(toString(tuple.4), '\'', '\\\'') || '\''
|
|
75
|
+
end) || ')'
|
|
76
|
+
))),
|
|
77
|
+
values_tuples
|
|
78
|
+
)
|
|
79
|
+
),
|
|
80
|
+
', '
|
|
81
|
+
) || ');\nINSERT INTO {table_dest}\nSELECT * FROM {table_temp};' as move_query
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
with
|
|
2
|
+
'{table_temp}' as table_temp
|
|
3
|
+
, '{table_dest}' as table_dest
|
|
4
|
+
, partitions as (
|
|
5
|
+
select partition
|
|
6
|
+
from system.parts
|
|
7
|
+
where database = splitByChar('.', table_temp)[1]
|
|
8
|
+
and table = splitByChar('.', table_temp)[2]
|
|
9
|
+
and partition <> 'tuple()'
|
|
10
|
+
and active
|
|
11
|
+
order by partition
|
|
12
|
+
)
|
|
13
|
+
select
|
|
14
|
+
toBool(count() > 0) as is_avaliable
|
|
15
|
+
, if(
|
|
16
|
+
is_avaliable
|
|
17
|
+
, 'alter table ' || table_dest || ' ' ||
|
|
18
|
+
arrayStringConcat(
|
|
19
|
+
arrayMap(
|
|
20
|
+
p -> 'replace partition ' || p || ' from ' || table_temp
|
|
21
|
+
, groupArray(partition)
|
|
22
|
+
), ', ')
|
|
23
|
+
, '') as move_query
|
|
24
|
+
from partitions
|