dbhose-airflow 0.0.2.0__py3-none-any.whl → 0.0.2.2__py3-none-any.whl

dbhose_airflow/dq/clickhouse/infinity.sql

@@ -0,0 +1,34 @@
+with
+    replaceAll('{table}', '`', '') as table_name
+  , splitByChar('.', table_name) as table_parts
+select
+    have_test
+  , column_name
+  , query
+from (
+    select
+        database
+      , table
+      , toBool(countIf(type like 'Float%')) as have_test
+    from system.columns
+    where database = table_parts[1]
+        and table = table_parts[2]
+    group by 1, 2
+) as ht
+left join (
+    select 
+        name as column_name
+      , format(
+            'select count(*) as value, if(count() = 0, ''Pass'', ''Fail'') as result \n' ||
+            'from {{}} where isInfinite({{}})'
+          , table_name
+          , column_name
+        ) as query
+      , database
+      , table
+    from system.columns
+    where database = table_parts[1]
+        and table = table_parts[2]
+        and type like 'Float%'
+) as qt
+using(database, table)

dbhose_airflow/dq/clickhouse/nan.sql

@@ -0,0 +1,34 @@
+with
+    replaceAll('{table}', '`', '') as table_name
+  , splitByChar('.', table_name) as table_parts
+select
+    have_test
+  , column_name
+  , query
+from (
+    select
+        database
+      , table
+      , toBool(countIf(type in ('Float32', 'Float64'))) as have_test
+    from system.columns
+    where database = table_parts[1]
+        and table = table_parts[2]
+    group by 1, 2
+) as ht
+left join (
+    select 
+        name as column_name
+      , format(
+            'select count(*) as value, if(count() = 0, ''Pass'', ''Fail'') as result \n' ||
+            'from {{}} where isNaN({{}})'
+          , table_name
+          , column_name
+        ) as query
+      , database
+      , table
+    from system.columns
+    where database = table_parts[1]
+        and table = table_parts[2]
+        and type in ('Float32', 'Float64')
+) as qt
+using(database, table)

dbhose_airflow/dq/clickhouse/sum.sql

@@ -0,0 +1,72 @@
+with
+    replaceAll('{table}', '`', '') as table_name
+  , splitByChar('.', table_name) as table_parts
+select
+    have_test
+  , column_name
+  , query
+from (
+    select
+        database
+      , table
+      , toBool(countIf(type in (
+            'Int8'
+          , 'Int16'
+          , 'Int32'
+          , 'Int64'
+          , 'Int128'
+          , 'Int256'
+          , 'UInt8'
+          , 'UInt16'
+          , 'UInt32'
+          , 'UInt64'
+          , 'UInt128'
+          , 'UInt256'
+          , 'Float32'
+          , 'Float64'
+          , 'Decimal32'
+          , 'Decimal64'
+          , 'Decimal128'
+          , 'Decimal256'
+        ))) as have_test
+    from system.columns
+    where database = table_parts[1]
+        and table = table_parts[2]
+    group by 1, 2
+) as ht
+left join (
+    select 
+        name as column_name
+      , format(
+            'select round(sum({{}}), 2) as value \n' ||
+            'from {{}}'
+          , column_name
+          , table_name
+        ) as query
+      , database
+      , table
+    from system.columns
+    where database = table_parts[1]
+        and table = table_parts[2]
+        and type in (
+            'Int8'
+          , 'Int16'
+          , 'Int32'
+          , 'Int64'
+          , 'Int128'
+          , 'Int256'
+          , 'UInt8'
+          , 'UInt16'
+          , 'UInt32'
+          , 'UInt64'
+          , 'UInt128'
+          , 'UInt256'
+          , 'Float32'
+          , 'Float64'
+          , 'Decimal32'
+          , 'Decimal64'
+          , 'Decimal128'
+          , 'Decimal256'
+        )
+) as qt
+using(database, table)

dbhose_airflow/dq/clickhouse/total.sql

@@ -0,0 +1 @@
+select count(*) as value from {table}

dbhose_airflow/dq/clickhouse/uniq.sql

@@ -0,0 +1 @@
+select count(*) - countDistinct(*) as value, case when value = 0 then 'Pass' else 'Fail' end as result from {table}

dbhose_airflow/dq/greenplum/empty.sql

@@ -0,0 +1 @@
+select count(*) as value, case when count(*) = 0 then 'Fail' else 'Pass' end as result from {table}

dbhose_airflow/dq/greenplum/future.sql

@@ -0,0 +1,55 @@
+with cte as (
+    select '{table}'::regclass as table_class
+)
+select
+    have_test
+  , column_name
+  , query
+from (
+    select 
+        n.nspname::text as schema_name
+      , c.relname::text as table_name
+      , count(*) filter(
+            where format_type(a.atttypid, a.atttypmod) in (
+                'date'
+              , 'timestamp with time zone'
+              , 'timestamp without time zone'
+            )
+        ) > 0 as have_test
+    from pg_class as c
+    join pg_namespace as n
+        on n.oid = c.relnamespace
+    join pg_attribute as a
+        on a.attrelid = c.oid
+    where c.oid = (select table_class from cte)
+        and a.attnum > 0
+        and not a.attisdropped
+    group by 1, 2
+) as ht
+left join (
+    select 
+        a.attname::text as column_name
+      , n.nspname::text as schema_name
+      , c.relname::text as table_name
+      , format(
+            e'select count(*) as value, case when count(*) = 0 then ''Pass'' else ''Fail'' end as result \n' ||
+            e'from %I.%I where %I > (current_date + interval ''1 month'')'
+          , n.nspname
+          , c.relname
+          , a.attname
+        ) as query
+    from pg_attribute as a
+    join pg_class as c
+        on a.attrelid = c.oid
+    join pg_namespace as n
+        on n.oid = c.relnamespace
+    where c.oid = (select table_class from cte)
+        and a.attnum > 0 
+        and not a.attisdropped
+        and format_type(a.atttypid, a.atttypmod) in (
+            'date'
+          , 'timestamp with time zone'
+          , 'timestamp without time zone'
+        )
+) as qt
+using(schema_name, table_name)

dbhose_airflow/dq/greenplum/infinity.sql

@@ -0,0 +1,60 @@
+with cte as (
+    select
+        '{table}'::regclass as table_class
+)
+select
+    have_test
+  , column_name
+  , query
+from (
+    select 
+        n.nspname::text as schema_name
+      , c.relname::text as table_name
+      , count(*) filter(
+            where format_type(a.atttypid, a.atttypmod) in (
+                'date'
+              , 'double precision'
+              , 'real'
+              , 'timestamp with time zone'
+              , 'timestamp without time zone'
+            )
+        ) > 0 as have_test
+    from pg_class as c
+    join pg_namespace as n
+        on n.oid = c.relnamespace
+    join pg_attribute as a
+        on a.attrelid = c.oid
+    where c.oid = (select table_class from cte)
+        and a.attnum > 0
+        and not a.attisdropped
+    group by 1, 2
+) as ht
+left join (
+    select 
+        a.attname::text as column_name
+      , n.nspname::text as schema_name
+      , c.relname::text as table_name
+      , format(
+            E'select count(*) as value, case when count(*) = 0 then ''Pass'' else ''Fail'' end as result \n' ||
+            E'from %I.%I where %I in (''infinity'', ''-infinity'')'
+          , n.nspname
+          , c.relname
+          , a.attname
+        ) as query
+    from pg_attribute as a
+    join pg_class as c
+        on a.attrelid = c.oid
+    join pg_namespace as n
+        on n.oid = c.relnamespace
+    where c.oid = (select table_class from cte)
+        and a.attnum > 0 
+        and not a.attisdropped
+        and format_type(a.atttypid, a.atttypmod) in (
+            'date'
+          , 'double precision'
+          , 'real'
+          , 'timestamp with time zone'
+          , 'timestamp without time zone'
+        )
+) as qt
+using(schema_name, table_name)

dbhose_airflow/dq/greenplum/nan.sql

@@ -0,0 +1,58 @@
+with cte as (
+    select
+        '{table}'::regclass as table_class
+)
+select
+    have_test
+  , column_name
+  , query
+from (
+    select 
+        n.nspname::text as schema_name
+      , c.relname::text as table_name
+      , count(*) filter(
+            where format_type(a.atttypid, a.atttypmod) in (
+                'decimal'
+              , 'double precision'
+              , 'real'
+              , 'numeric'
+            )
+        ) > 0 as have_test
+    from pg_class as c
+    join pg_namespace as n
+        on n.oid = c.relnamespace
+    join pg_attribute as a
+        on a.attrelid = c.oid
+    where c.oid = (select table_class from cte)
+        and a.attnum > 0
+        and not a.attisdropped
+    group by 1, 2
+) as ht
+left join (
+    select 
+        a.attname::text as column_name
+      , n.nspname::text as schema_name
+      , c.relname::text as table_name
+      , format(
+            E'select count(*) as value, case when count(*) = 0 then ''Pass'' else ''Fail'' end as result \n' ||
+            E'from %I.%I where %I = ''nan'''
+          , n.nspname
+          , c.relname
+          , a.attname
+        ) as query
+    from pg_attribute as a
+    join pg_class as c
+        on a.attrelid = c.oid
+    join pg_namespace as n
+        on n.oid = c.relnamespace
+    where c.oid = (select table_class from cte)
+        and a.attnum > 0 
+        and not a.attisdropped
+        and format_type(a.atttypid, a.atttypmod) in (
+            'decimal'
+          , 'double precision'
+          , 'real'
+          , 'numeric'
+        )
+) as qt
+using(schema_name, table_name)

dbhose_airflow/dq/greenplum/sum.sql

@@ -0,0 +1,63 @@
+with cte as (
+    select '{table}'::regclass as table_class
+)
+select
+    have_test
+  , column_name
+  , query
+from (
+    select 
+        n.nspname::text as schema_name
+      , c.relname::text as table_name
+      , count(*) filter(
+            where format_type(a.atttypid, a.atttypmod) in (
+                'smallint'
+              , 'integer'
+              , 'bigint'
+              , 'decimal'
+              , 'numeric'
+              , 'real'
+              , 'double precision'
+            )
+        ) > 0 as have_test
+    from pg_class as c
+    join pg_namespace as n
+        on n.oid = c.relnamespace
+    join pg_attribute as a
+        on a.attrelid = c.oid
+    where c.oid = (select table_class from cte)
+        and a.attnum > 0
+        and not a.attisdropped
+    group by 1, 2
+) as ht
+left join (
+    select 
+        a.attname::text as column_name
+      , n.nspname::text as schema_name
+      , c.relname::text as table_name
+      , format(
+            E'select round(sum(coalesce(%I::numeric, 0)), 2) as value \n' ||
+            E'from %I.%I'
+          , a.attname
+          , n.nspname
+          , c.relname
+        ) as query
+    from pg_attribute as a
+    join pg_class as c
+        on a.attrelid = c.oid
+    join pg_namespace as n
+        on n.oid = c.relnamespace
+    where c.oid = (select table_class from cte)
+        and a.attnum > 0 
+        and not a.attisdropped
+        and format_type(a.atttypid, a.atttypmod) in (
+            'smallint'
+          , 'integer'
+          , 'bigint'
+          , 'decimal'
+          , 'numeric'
+          , 'real'
+          , 'double precision'
+        )
+) as qt
+using(schema_name, table_name)

dbhose_airflow/dq/greenplum/total.sql

@@ -0,0 +1 @@
+select count(*) as value from {table}

dbhose_airflow/dq/greenplum/uniq.sql

@@ -0,0 +1,2 @@
+with cte(cnt, uniq) as (select (select count(*) from {table}), (select count(*) from (select distinct * from {table}) as uniq))
+select cnt - uniq as value, case when cnt - uniq = 0 then 'Pass' else 'Fail' end as result from cte

dbhose_airflow/dq/postgres/empty.sql

@@ -0,0 +1 @@
+select count(*) as value, case when count(*) = 0 then 'Fail' else 'Pass' end as result from {table}

dbhose_airflow/dq/postgres/future.sql

@@ -0,0 +1,55 @@
+with cte as (
+    select '{table}'::regclass as table_class
+)
+select
+    have_test
+  , column_name
+  , query
+from (
+    select 
+        n.nspname::text as schema_name
+      , c.relname::text as table_name
+      , count(*) filter(
+            where format_type(a.atttypid, a.atttypmod) in (
+                'date'
+              , 'timestamp with time zone'
+              , 'timestamp without time zone'
+            )
+        ) > 0 as have_test
+    from pg_class as c
+    join pg_namespace as n
+        on n.oid = c.relnamespace
+    join pg_attribute as a
+        on a.attrelid = c.oid
+    where c.oid = (select table_class from cte)
+        and a.attnum > 0
+        and not a.attisdropped
+    group by 1, 2
+) as ht
+left join (
+    select 
+        a.attname::text as column_name
+      , n.nspname::text as schema_name
+      , c.relname::text as table_name
+      , format(
+            e'select count(*) as value, case when count(*) = 0 then ''Pass'' else ''Fail'' end as result \n' ||
+            e'from %I.%I where %I > (current_date + interval ''1 month'')'
+          , n.nspname
+          , c.relname
+          , a.attname
+        ) as query
+    from pg_attribute as a
+    join pg_class as c
+        on a.attrelid = c.oid
+    join pg_namespace as n
+        on n.oid = c.relnamespace
+    where c.oid = (select table_class from cte)
+        and a.attnum > 0 
+        and not a.attisdropped
+        and format_type(a.atttypid, a.atttypmod) in (
+            'date'
+          , 'timestamp with time zone'
+          , 'timestamp without time zone'
+        )
+) as qt
+using(schema_name, table_name)

dbhose_airflow/dq/postgres/infinity.sql

@@ -0,0 +1,60 @@
+with cte as (
+    select
+        '{table}'::regclass as table_class
+)
+select
+    have_test
+  , column_name
+  , query
+from (
+    select 
+        n.nspname::text as schema_name
+      , c.relname::text as table_name
+      , count(*) filter(
+            where format_type(a.atttypid, a.atttypmod) in (
+                'date'
+              , 'double precision'
+              , 'real'
+              , 'timestamp with time zone'
+              , 'timestamp without time zone'
+            )
+        ) > 0 as have_test
+    from pg_class as c
+    join pg_namespace as n
+        on n.oid = c.relnamespace
+    join pg_attribute as a
+        on a.attrelid = c.oid
+    where c.oid = (select table_class from cte)
+        and a.attnum > 0
+        and not a.attisdropped
+    group by 1, 2
+) as ht
+left join (
+    select 
+        a.attname::text as column_name
+      , n.nspname::text as schema_name
+      , c.relname::text as table_name
+      , format(
+            E'select count(*) as value, case when count(*) = 0 then ''Pass'' else ''Fail'' end as result \n' ||
+            E'from %I.%I where %I in (''infinity'', ''-infinity'')'
+          , n.nspname
+          , c.relname
+          , a.attname
+        ) as query
+    from pg_attribute as a
+    join pg_class as c
+        on a.attrelid = c.oid
+    join pg_namespace as n
+        on n.oid = c.relnamespace
+    where c.oid = (select table_class from cte)
+        and a.attnum > 0 
+        and not a.attisdropped
+        and format_type(a.atttypid, a.atttypmod) in (
+            'date'
+          , 'double precision'
+          , 'real'
+          , 'timestamp with time zone'
+          , 'timestamp without time zone'
+        )
+) as qt
+using(schema_name, table_name)

dbhose_airflow/dq/postgres/nan.sql

@@ -0,0 +1,58 @@
+with cte as (
+    select
+        '{table}'::regclass as table_class
+)
+select
+    have_test
+  , column_name
+  , query
+from (
+    select 
+        n.nspname::text as schema_name
+      , c.relname::text as table_name
+      , count(*) filter(
+            where format_type(a.atttypid, a.atttypmod) in (
+                'decimal'
+              , 'double precision'
+              , 'real'
+              , 'numeric'
+            )
+        ) > 0 as have_test
+    from pg_class as c
+    join pg_namespace as n
+        on n.oid = c.relnamespace
+    join pg_attribute as a
+        on a.attrelid = c.oid
+    where c.oid = (select table_class from cte)
+        and a.attnum > 0
+        and not a.attisdropped
+    group by 1, 2
+) as ht
+left join (
+    select 
+        a.attname::text as column_name
+      , n.nspname::text as schema_name
+      , c.relname::text as table_name
+      , format(
+            E'select count(*) as value, case when count(*) = 0 then ''Pass'' else ''Fail'' end as result \n' ||
+            E'from %I.%I where %I = ''nan'''
+          , n.nspname
+          , c.relname
+          , a.attname
+        ) as query
+    from pg_attribute as a
+    join pg_class as c
+        on a.attrelid = c.oid
+    join pg_namespace as n
+        on n.oid = c.relnamespace
+    where c.oid = (select table_class from cte)
+        and a.attnum > 0 
+        and not a.attisdropped
+        and format_type(a.atttypid, a.atttypmod) in (
+            'decimal'
+          , 'double precision'
+          , 'real'
+          , 'numeric'
+        )
+) as qt
+using(schema_name, table_name)

dbhose_airflow/dq/postgres/sum.sql

@@ -0,0 +1,63 @@
+with cte as (
+    select '{table}'::regclass as table_class
+)
+select
+    have_test
+  , column_name
+  , query
+from (
+    select 
+        n.nspname::text as schema_name
+      , c.relname::text as table_name
+      , count(*) filter(
+            where format_type(a.atttypid, a.atttypmod) in (
+                'smallint'
+              , 'integer'
+              , 'bigint'
+              , 'decimal'
+              , 'numeric'
+              , 'real'
+              , 'double precision'
+            )
+        ) > 0 as have_test
+    from pg_class as c
+    join pg_namespace as n
+        on n.oid = c.relnamespace
+    join pg_attribute as a
+        on a.attrelid = c.oid
+    where c.oid = (select table_class from cte)
+        and a.attnum > 0
+        and not a.attisdropped
+    group by 1, 2
+) as ht
+left join (
+    select 
+        a.attname::text as column_name
+      , n.nspname::text as schema_name
+      , c.relname::text as table_name
+      , format(
+            E'select round(sum(coalesce(%I::numeric, 0)), 2) as value \n' ||
+            E'from %I.%I'
+          , a.attname
+          , n.nspname
+          , c.relname
+        ) as query
+    from pg_attribute as a
+    join pg_class as c
+        on a.attrelid = c.oid
+    join pg_namespace as n
+        on n.oid = c.relnamespace
+    where c.oid = (select table_class from cte)
+        and a.attnum > 0 
+        and not a.attisdropped
+        and format_type(a.atttypid, a.atttypmod) in (
+            'smallint'
+          , 'integer'
+          , 'bigint'
+          , 'decimal'
+          , 'numeric'
+          , 'real'
+          , 'double precision'
+        )
+) as qt
+using(schema_name, table_name)

dbhose_airflow/dq/postgres/total.sql

@@ -0,0 +1 @@
+select count(*) as value from {table}

dbhose_airflow/dq/postgres/uniq.sql

@@ -0,0 +1,2 @@
+with cte(cnt, uniq) as (select (select count(*) from {table}), (select count(*) from (select distinct * from {table}) as uniq))
+select cnt - uniq as value, case when cnt - uniq = 0 then 'Pass' else 'Fail' end as result from cte