dbhose-airflow 0.0.2.0__py3-none-any.whl → 0.0.2.2__py3-none-any.whl

dbhose_airflow/LOGO

@@ -0,0 +1,9 @@
+
+                                                                 (                )
+ (  (                                                 )          )\ )     (    ( /(
+ )\))(   '   (    (                   )       (     ( /(        (()/(   ( )\   )\())               (
+((_)()\ )   ))\   )\    (     (      (       ))\    )\())   (    /(_))  )((_) ((_)\    (    (     ))\
+_(())\_)() /((_) (( )   )\    )\     )\  '  /((_)  (_))/    )\   (_))_  ((_)_   _((_)   )\   )\   /((_)
+\ \((_)/ /(_))   | |   ((_)  ((_)  _((_))  (_))    | |_    ((_)   |   \  | _ ) | || |  ((_) ((_) (_))
+ \ \/\/ / / -_)  | |  / _|  / _ \ | '  \() / -_)   |  _|  / _ \   | |) | | _ \ | __ | / _ \ (_-< / -_)
+  \_/\_/  \___|  |_|  \__|  \___/ |_|_|_|  \___|    \__|  \___/   |___/  |___/ |_||_| \___/ /__/ \___|

dbhose_airflow/__init__.py

@@ -14,6 +14,7 @@ from pandas import DataFrame as PDFrame
 from polars import DataFrame as PLFrame
 
 from .airflow_connect import dbhose_dumper
+from .chunk_query import query_part
 from .dq_check import DQCheck
 from .move_method import MoveMethod
 
@@ -31,7 +32,7 @@ __all__ = (
     "dbhose_dumper",
 )
 __author__ = "0xMihalich"
-__version__ = "0.0.2.0"
+__version__ = "0.0.2.2"
 
 
 root_path = dirname(__file__)
@@ -104,7 +105,7 @@ class DBHose:
         self.connection_dest = connection_dest
         self.connection_src = connection_src
         self.dq_skip_check = dq_skip_check
-        self.filter_by = filter_by
+        self.filter_by = ", ".join(filter_by)
         self.drop_temp_table = drop_temp_table
         self.move_method = move_method
         self.custom_move = custom_move
@@ -333,12 +334,23 @@ class DBHose:
                 self.logger.error(wrap_frame(error_msg))
                 raise ValueError(error_msg)
 
-            self.dumper_dest.cursor.execute(self.custom_move)
+            for query in query_part(self.custom_move):
+                self.dumper_dest.cursor.execute(query)
 
             if self.dumper_dest.__class__ is not NativeDumper:
                 self.dumper_dest.connect.commit()
 
         elif self.move_method.have_sql:
+
+            if (
+                self.move_method is MoveMethod.delete
+                and self.dumper_dest.__class__ is NativeDumper
+                and len(self.filter_by.split(", ")) > 4
+            ):
+                error_msg = "Too many columns in filter_by (> 4)"
+                self.logger.error(wrap_frame(error_msg))
+                raise ValueError(error_msg)
+
             move_query = read_text(
                 mv_path.format(self.dumper_dest.dbname, self.move_method.name)
             )
@@ -357,12 +369,24 @@ class DBHose:
                 self.logger.error(wrap_frame(error_msg))
                 raise ValueError(error_msg)
 
-            self.dumper_dest.cursor.execute(move_query)
+            for query in query_part(move_query):
+                self.dumper_dest.cursor.execute(query)
 
             if self.dumper_dest.__class__ is not NativeDumper:
                 self.dumper_dest.connect.commit()
 
         else:
+            if self.move_method is MoveMethod.rewrite:
+                self.logger.info("Clear table operation start")
+                self.dumper_dest.cursor.execute(
+                    f"truncate table {self.table_dest}"
+                )
+
+                if self.dumper_dest.__class__ is not NativeDumper:
+                    self.dumper_dest.connect.commit()
+
+                self.logger.info("Clear table operation done")
+
             self.dumper_dest.write_between(self.table_dest, self.table_temp)
 
         self.logger.info(wrap_frame(f"Data moved into {self.table_dest}"))

dbhose_airflow/chunk_query.py

@@ -0,0 +1,14 @@
+from re import split
+
+
+pattern = r";(?=(?:[^']*'[^']*')*[^']*$)"
+
+
+def query_part(query: str) -> tuple[str]:
+    """Chunk multiquery to parts."""
+
+    return (
+        part.strip(";").strip()
+        for part in split(pattern, query)
+        if part.strip(";").strip()
+    )

dbhose_airflow/ddl/clickhouse.sql

@@ -0,0 +1,81 @@
+WITH
+    split_table AS (
+        SELECT 
+            splitByChar('.', '{table}')[1] as db,
+            splitByChar('.', '{table}')[2] as tbl
+    ),
+    original_ddl AS (
+        SELECT create_table_query
+        FROM system.tables 
+        WHERE database = (SELECT db FROM split_table)
+          AND name = (SELECT tbl FROM split_table)
+    ),
+    columns_data AS (
+        SELECT 
+            groupArray(
+                tuple(name, type, default_expression, comment)
+            ) as cols
+        FROM (
+            SELECT 
+                name,
+                type,
+                default_expression,
+                comment
+            FROM system.columns 
+            WHERE database = (SELECT db FROM split_table)
+              AND table = (SELECT tbl FROM split_table)
+            ORDER BY position
+        )
+    ),
+    table_info AS (
+        SELECT 
+            partition_key,
+            sorting_key,
+            primary_key
+        FROM system.tables 
+        WHERE database = (SELECT db FROM split_table)
+          AND name = (SELECT tbl FROM split_table)
+    ),
+    order_by_clause AS (
+        SELECT 
+            if((SELECT sorting_key FROM table_info) != '', 
+               concat('ORDER BY (', (SELECT sorting_key FROM table_info), ')'),
+               if((SELECT primary_key FROM table_info) != '',
+                  concat('ORDER BY (', (SELECT primary_key FROM table_info), ')'),
+                  'ORDER BY tuple()'
+               )
+            ) as order_by
+    )
+SELECT
+    (SELECT create_table_query FROM original_ddl) as ddl,
+    concat(
+        'CREATE TABLE IF NOT EXISTS ',
+        (SELECT db FROM split_table),
+        '.',
+        (SELECT tbl FROM split_table),
+        '_temp\n(\n',
+        arrayStringConcat(
+            arrayMap(
+                c -> concat('    ', c.1, ' ', c.2,
+                           if(c.3 != '', concat(' DEFAULT ', c.3), ''),
+                           if(c.4 != '', concat(' COMMENT ''', c.4, ''''), '')
+                ),
+                (SELECT cols FROM columns_data)
+            ),
+            ',' || '\n'
+        ),
+        '\n)\nENGINE = MergeTree()',
+        if((SELECT partition_key FROM table_info) != '', 
+           concat('\nPARTITION BY ', (SELECT partition_key FROM table_info)), 
+           ''),
+        if((SELECT order_by FROM order_by_clause) != '', 
+           concat('\n', (SELECT order_by FROM order_by_clause)), 
+           ''),
+        '\nSETTINGS index_granularity = 8192'
+    ) as temp_ddl,
+    concat(
+        (SELECT db FROM split_table),
+        '.',
+        (SELECT tbl FROM split_table),
+        '_temp'
+    ) as table_temp

dbhose_airflow/ddl/greenplum.sql

@@ -0,0 +1,229 @@
+WITH table_info AS (
+    SELECT 
+        CASE
+            WHEN c.relkind = 'r' THEN 'TABLE'
+            WHEN c.relkind = 'p' THEN 'TABLE'
+            WHEN c.relkind = 'u' THEN 'UNLOGGED TABLE'
+            WHEN c.relkind = 'v' THEN 'VIEW'
+            WHEN c.relkind = 'm' THEN 'MATERIALIZED VIEW'
+            ELSE 'UNKNOWN'
+        END as object_type,
+        quote_ident(n.nspname) as schema_name,
+        quote_ident(c.relname) as table_name,
+        pg_get_partkeydef(c.oid) as partition_by,
+        col_description(c.oid, 0) as table_comment,
+        am.amname as storage_type,
+        c.reloptions as with_options,
+        c.oid as table_oid,
+        c.relkind as relkind,
+        pg_get_viewdef(c.oid) as view_definition
+    FROM pg_class as c
+    JOIN pg_namespace as n ON c.relnamespace = n.oid
+    LEFT JOIN pg_am as am ON c.relam = am.oid
+    WHERE c.oid = '{table}'::regclass
+),
+partitions_info AS (
+    SELECT
+        child.relname as table_partition,
+        pg_get_expr(child.relpartbound, child.oid) as partition_constraint
+    FROM pg_inherits
+    JOIN pg_class child ON pg_inherits.inhrelid = child.oid
+    WHERE pg_inherits.inhparent = '{table}'::regclass
+),
+partitions_ddl AS (
+    SELECT 
+        string_agg(
+            'CREATE TABLE IF NOT EXISTS ' || (SELECT schema_name FROM table_info) || '.' ||
+            CASE
+                WHEN (SELECT table_name FROM table_info) ~ '^".*"$' THEN
+                    '"' || regexp_replace(table_partition, '^"|"$', '', 'g') || '_temp"'
+                ELSE
+                    table_partition || '_temp'
+            END || 
+            ' PARTITION OF ' || (SELECT schema_name FROM table_info) || '.' || 
+            CASE
+                WHEN (SELECT table_name FROM table_info) ~ '^".*"$' THEN
+                    '"' || regexp_replace((SELECT table_name FROM table_info), '^"|"$', '', 'g') || '_temp"'
+                ELSE
+                    (SELECT table_name FROM table_info) || '_temp'
+            END || ' ' || partition_constraint || ';',
+            E'\n'
+        ) as ddl
+    FROM partitions_info
+),
+columns_info AS (
+    SELECT
+        quote_ident(pa.attname) as column_name,
+        format_type(pa.atttypid, pa.atttypmod) as data_type,
+        pg_get_expr(ad.adbin, ad.adrelid) as default_value,
+        pa.attnotnull as not_null,
+        col_description(pa.attrelid, pa.attnum) as column_comment,
+        pa.attnum as column_order
+    FROM pg_attribute as pa
+    LEFT JOIN pg_attrdef as ad ON pa.attrelid = ad.adrelid AND pa.attnum = ad.adnum
+    WHERE pa.attrelid = '{table}'::regclass
+        AND pa.attnum > 0
+        AND NOT pa.attisdropped
+),
+constraints_info AS (
+    SELECT DISTINCT
+        conname as constraint_name,
+        pg_get_constraintdef(oid) as constraint_definition
+    FROM pg_constraint
+    WHERE conrelid = '{table}'::regclass
+),
+constraints_list AS (
+    SELECT 
+        string_agg(
+            '    CONSTRAINT ' || quote_ident(constraint_name) || ' ' || constraint_definition,
+            ',' || E'\n'
+        ) as constraints_ddl
+    FROM constraints_info
+),
+indexes_info AS (
+    SELECT
+        indexname,
+        indexdef
+    FROM pg_indexes 
+    WHERE schemaname = 'mart'
+        AND tablename = 'dim_branch'
+),
+indexes_ddl AS (
+    SELECT 
+        string_agg(indexdef || ';', E'\n') as ddl
+    FROM indexes_info
+),
+distribution_info AS (
+    SELECT
+        CASE 
+            WHEN policytype = 'r' THEN 'DISTRIBUTED RANDOMLY'
+            WHEN policytype = 'p' THEN 'DISTRIBUTED BY (' || 
+                (SELECT string_agg(quote_ident(a.attname), ', ' ORDER BY pos.ordinality)
+                 FROM gp_distribution_policy p2
+                 CROSS JOIN LATERAL unnest(p2.distkey) WITH ORDINALITY AS pos(attnum, ordinality)
+                 JOIN pg_attribute a ON a.attrelid = p2.localoid AND a.attnum = pos.attnum
+                 WHERE p2.localoid = '{table}'::regclass) || ')'
+            ELSE 'DISTRIBUTED UNKNOWN'
+        END as distribution_clause
+    FROM gp_distribution_policy
+    WHERE localoid = '{table}'::regclass
+),
+table_ddl AS (
+    SELECT 
+        'CREATE ' || 
+        CASE 
+            WHEN (SELECT storage_type FROM table_info) = 'heap' THEN '' 
+            ELSE 'UNLOGGED ' 
+        END ||
+        'TABLE ' || (SELECT schema_name FROM table_info) || '.' || (SELECT table_name FROM table_info) || ' (' || E'\n' ||
+        string_agg(
+            '    ' || ci.column_name || ' ' || ci.data_type ||
+            CASE WHEN ci.default_value IS NOT NULL THEN ' DEFAULT ' || ci.default_value ELSE '' END ||
+            CASE WHEN ci.not_null THEN ' NOT NULL' ELSE '' END,
+            ',' || E'\n' ORDER BY ci.column_order
+        ) ||
+        CASE 
+            WHEN (SELECT constraints_ddl FROM constraints_list) IS NOT NULL THEN 
+                ',' || E'\n' || (SELECT constraints_ddl FROM constraints_list)
+            ELSE ''
+        END ||
+        E'\n' || ')' ||
+        CASE WHEN (SELECT partition_by FROM table_info) IS NOT NULL THEN E'\nPARTITION BY ' || (SELECT partition_by FROM table_info) ELSE '' END ||
+        CASE WHEN (SELECT storage_type FROM table_info) IS NOT NULL THEN E'\n' || 'USING ' || (SELECT storage_type FROM table_info) ELSE '' END ||
+        CASE WHEN (SELECT with_options FROM table_info) IS NOT NULL THEN E'\n' || 'WITH (' || array_to_string((SELECT with_options FROM table_info), ', ') || ')' ELSE '' END ||
+        E'\n' || (SELECT distribution_clause FROM distribution_info) || ';' as ddl
+    FROM columns_info ci
+    WHERE (SELECT relkind FROM table_info) IN ('r', 'p', 'u')
+),
+view_ddl AS (
+    SELECT 
+        'CREATE VIEW ' || (SELECT schema_name FROM table_info) || '.' || (SELECT table_name FROM table_info) || 
+        ' AS ' || (SELECT view_definition FROM table_info) as ddl
+    WHERE (SELECT relkind FROM table_info) = 'v'
+),
+matview_ddl AS (
+    SELECT 
+        'CREATE MATERIALIZED VIEW ' || (SELECT schema_name FROM table_info) || '.' || (SELECT table_name FROM table_info) || 
+        ' AS ' || (SELECT view_definition FROM table_info) || 
+        CASE WHEN (SELECT distribution_clause FROM distribution_info) IS NOT NULL THEN 
+            E'\n' || (SELECT distribution_clause FROM distribution_info)
+        ELSE '' END as ddl
+    WHERE (SELECT relkind FROM table_info) = 'm'
+),
+column_comments_ddl AS (
+    SELECT 
+        string_agg(
+            'COMMENT ON COLUMN ' || (SELECT schema_name FROM table_info) || '.' || (SELECT table_name FROM table_info) || 
+            '.' || column_name || ' IS ' || quote_literal(column_comment) || ';',
+            E'\n'
+        ) as ddl
+    FROM columns_info
+    WHERE column_comment IS NOT NULL
+),
+table_comment_ddl AS (
+    SELECT 
+        CASE 
+            WHEN (SELECT table_comment FROM table_info) IS NOT NULL THEN
+                'COMMENT ON TABLE ' || (SELECT schema_name FROM table_info) || '.' || (SELECT table_name FROM table_info) || 
+                ' IS ' || quote_literal((SELECT table_comment FROM table_info)) || ';'
+            ELSE ''
+        END as ddl
+),
+temp_table_ddl AS (
+    SELECT 
+        'CREATE UNLOGGED TABLE IF NOT EXISTS ' || (SELECT schema_name FROM table_info) || '.' || 
+        CASE
+            WHEN (SELECT table_name FROM table_info) ~ '^".*"$' THEN
+                '"' || regexp_replace((SELECT table_name FROM table_info), '^"|"$', '', 'g') || '_temp"'
+            ELSE
+                (SELECT table_name FROM table_info) || '_temp'
+        END || ' (' || E'\n' ||
+        string_agg(
+            '    ' || ci.column_name || ' ' || ci.data_type ||
+            CASE WHEN ci.default_value IS NOT NULL THEN ' DEFAULT ' || ci.default_value ELSE '' END ||
+            CASE WHEN ci.not_null THEN ' NOT NULL' ELSE '' END,
+            ',' || E'\n' ORDER BY ci.column_order
+        ) || E'\n' || ')' ||
+        CASE WHEN (SELECT partition_by FROM table_info) IS NOT NULL THEN 
+            E'\nPARTITION BY ' || (SELECT partition_by FROM table_info)
+        ELSE '' END || ';' ||
+        CASE 
+            WHEN (SELECT ddl FROM partitions_ddl) IS NOT NULL THEN 
+                E'\n' || E'\n' || (SELECT ddl FROM partitions_ddl)
+            ELSE ''
+        END as ddl
+    FROM columns_info ci
+),
+final_original_ddl AS (
+    SELECT 
+        COALESCE(
+            (SELECT ddl FROM table_ddl),
+            (SELECT ddl FROM view_ddl),
+            (SELECT ddl FROM matview_ddl)
+        ) ||
+        CASE 
+            WHEN (SELECT ddl FROM indexes_ddl) IS NOT NULL THEN 
+                E'\n' || (SELECT ddl FROM indexes_ddl)
+            ELSE ''
+        END ||
+        CASE 
+            WHEN (SELECT ddl FROM table_comment_ddl) != '' THEN 
+                E'\n' || (SELECT ddl FROM table_comment_ddl)
+            ELSE ''
+        END ||
+        CASE 
+            WHEN (SELECT ddl FROM column_comments_ddl) IS NOT NULL THEN 
+                E'\n' || (SELECT ddl FROM column_comments_ddl)
+            ELSE ''
+        END as original_ddl
+)
+SELECT
+    (SELECT original_ddl FROM final_original_ddl) as ddl,
+    (SELECT ddl FROM temp_table_ddl) as temp_ddl,
+    (SELECT schema_name FROM table_info) || '.' || 
+        CASE
+            WHEN (SELECT table_name FROM table_info) ~ '^".*"$' THEN
+                '"' || regexp_replace((SELECT table_name FROM table_info), '^"|"$', '', 'g') || '_temp"'
+            ELSE
+                (SELECT table_name FROM table_info) || '_temp'
+        END as table_temp

dbhose_airflow/ddl/postgres.sql

@@ -0,0 +1,209 @@
+WITH table_info AS (
+    SELECT 
+        CASE
+            WHEN c.relkind = 'r' THEN 'TABLE'
+            WHEN c.relkind = 'p' THEN 'TABLE'
+            WHEN c.relkind = 'u' THEN 'UNLOGGED TABLE'
+            WHEN c.relkind = 'v' THEN 'VIEW'
+            WHEN c.relkind = 'm' THEN 'MATERIALIZED VIEW'
+            ELSE 'UNKNOWN'
+        END as object_type,
+        quote_ident(n.nspname) as schema_name,
+        quote_ident(c.relname) as table_name,
+        pg_get_partkeydef(c.oid) as partition_by,
+        am.amname as storage_type,
+        c.reloptions as with_options,
+        c.oid as table_oid,
+        c.relkind as relkind,
+        col_description(c.oid, 0) as table_comment,
+        pg_get_viewdef(c.oid) as view_definition
+    FROM pg_class as c
+    JOIN pg_namespace as n ON c.relnamespace = n.oid
+    LEFT JOIN pg_am as am ON c.relam = am.oid
+    WHERE c.oid = '{table}'::regclass
+),
+partitions_info AS (
+    SELECT
+        child.relname as table_partition,
+        pg_get_expr(child.relpartbound, child.oid) as partition_constraint
+    FROM pg_inherits
+    JOIN pg_class child ON pg_inherits.inhrelid = child.oid
+    WHERE pg_inherits.inhparent = '{table}'::regclass
+),
+partitions_ddl AS (
+    SELECT 
+        string_agg(
+            'CREATE TABLE IF NOT EXISTS ' || (SELECT schema_name FROM table_info) || '.' ||
+            CASE
+                WHEN (SELECT table_name FROM table_info) ~ '^".*"$' THEN
+                    '"' || regexp_replace(table_partition, '^"|"$', '', 'g') || '_temp"'
+                ELSE
+                    table_partition || '_temp'
+            END || 
+            ' PARTITION OF ' || (SELECT schema_name FROM table_info) || '.' || 
+            CASE
+                WHEN (SELECT table_name FROM table_info) ~ '^".*"$' THEN
+                    '"' || regexp_replace((SELECT table_name FROM table_info), '^"|"$', '', 'g') || '_temp"'
+                ELSE
+                    (SELECT table_name FROM table_info) || '_temp'
+            END || ' ' || partition_constraint || ';',
+            E'\n'
+        ) as ddl
+    FROM partitions_info
+),
+columns_info AS (
+    SELECT
+        quote_ident(pa.attname) as column_name,
+        format_type(pa.atttypid, pa.atttypmod) as data_type,
+        pg_get_expr(ad.adbin, ad.adrelid) as default_value,
+        pa.attnotnull as not_null,
+        col_description(pa.attrelid, pa.attnum) as column_comment,
+        pa.attnum as column_order
+    FROM pg_attribute as pa
+    LEFT JOIN pg_attrdef as ad ON pa.attrelid = ad.adrelid AND pa.attnum = ad.adnum
+    WHERE pa.attrelid = '{table}'::regclass
+        AND pa.attnum > 0
+        AND NOT pa.attisdropped
+),
+constraints_info AS (
+    SELECT DISTINCT
+        conname as constraint_name,
+        pg_get_constraintdef(oid) as constraint_definition
+    FROM pg_constraint
+    WHERE conrelid = '{table}'::regclass
+),
+constraints_list AS (
+    SELECT 
+        string_agg(
+            '    CONSTRAINT ' || quote_ident(constraint_name) || ' ' || constraint_definition,
+            ',' || E'\n'
+        ) as constraints_ddl
+    FROM constraints_info
+),
+indexes_info AS (
+    SELECT
+        indexname,
+        indexdef
+    FROM pg_indexes 
+    WHERE schemaname = 'mart'
+        AND tablename = 'dim_branch'
+),
+indexes_ddl AS (
+    SELECT 
+        string_agg(indexdef || ';', E'\n') as ddl
+    FROM indexes_info
+),
+table_ddl AS (
+    SELECT 
+        'CREATE ' || 
+        CASE 
+            WHEN (SELECT storage_type FROM table_info) = 'heap' THEN '' 
+            ELSE 'UNLOGGED ' 
+        END ||
+        'TABLE ' || (SELECT schema_name FROM table_info) || '.' || (SELECT table_name FROM table_info) || ' (' || E'\n' ||
+        string_agg(
+            '    ' || ci.column_name || ' ' || ci.data_type ||
+            CASE WHEN ci.default_value IS NOT NULL THEN ' DEFAULT ' || ci.default_value ELSE '' END ||
+            CASE WHEN ci.not_null THEN ' NOT NULL' ELSE '' END,
+            ',' || E'\n' ORDER BY ci.column_order
+        ) ||
+        CASE 
+            WHEN (SELECT constraints_ddl FROM constraints_list) IS NOT NULL THEN 
+                ',' || E'\n' || (SELECT constraints_ddl FROM constraints_list)
+            ELSE ''
+        END ||
+        E'\n' || ')' ||
+        CASE WHEN (SELECT storage_type FROM table_info) IS NOT NULL AND (SELECT storage_type FROM table_info) != 'heap' THEN E'\n' || 'USING ' || (SELECT storage_type FROM table_info) ELSE '' END ||
+        CASE WHEN (SELECT with_options FROM table_info) IS NOT NULL THEN E'\n' || 'WITH (' || array_to_string((SELECT with_options FROM table_info), ', ') || ')' ELSE '' END || ';' as ddl
+    FROM columns_info ci
+    WHERE (SELECT relkind FROM table_info) IN ('r', 'p', 'u')
+),
+view_ddl AS (
+    SELECT 
+        'CREATE VIEW ' || (SELECT schema_name FROM table_info) || '.' || (SELECT table_name FROM table_info) || 
+        ' AS ' || (SELECT view_definition FROM table_info) as ddl
+    WHERE (SELECT relkind FROM table_info) = 'v'
+),
+matview_ddl AS (
+    SELECT 
+        'CREATE MATERIALIZED VIEW ' || (SELECT schema_name FROM table_info) || '.' || (SELECT table_name FROM table_info) || 
+        ' AS ' || (SELECT view_definition FROM table_info) as ddl
+    WHERE (SELECT relkind FROM table_info) = 'm'
+),
+column_comments_ddl AS (
+    SELECT 
+        string_agg(
+            'COMMENT ON COLUMN ' || (SELECT schema_name FROM table_info) || '.' || (SELECT table_name FROM table_info) || 
+            '.' || column_name || ' IS ' || quote_literal(column_comment) || ';',
+            E'\n'
+        ) as ddl
+    FROM columns_info
+    WHERE column_comment IS NOT NULL
+),
+table_comment_ddl AS (
+    SELECT 
+        CASE 
+            WHEN (SELECT table_comment FROM table_info) IS NOT NULL THEN
+                'COMMENT ON TABLE ' || (SELECT schema_name FROM table_info) || '.' || (SELECT table_name FROM table_info) || 
+                ' IS ' || quote_literal((SELECT table_comment FROM table_info)) || ';'
+            ELSE ''
+        END as ddl
+),
+temp_table_ddl AS (
+    SELECT 
+        'CREATE UNLOGGED TABLE IF NOT EXISTS ' || (SELECT schema_name FROM table_info) || '.' || 
+        CASE
+            WHEN (SELECT table_name FROM table_info) ~ '^".*"$' THEN
+                '"' || regexp_replace((SELECT table_name FROM table_info), '^"|"$', '', 'g') || '_temp"'
+            ELSE
+                (SELECT table_name FROM table_info) || '_temp'
+        END || ' (' || E'\n' ||
+        string_agg(
+            '    ' || ci.column_name || ' ' || ci.data_type ||
+            CASE WHEN ci.default_value IS NOT NULL THEN ' DEFAULT ' || ci.default_value ELSE '' END ||
+            CASE WHEN ci.not_null THEN ' NOT NULL' ELSE '' END,
+            ',' || E'\n' ORDER BY ci.column_order
+        ) || E'\n' || ')' ||
+        CASE WHEN (SELECT partition_by FROM table_info) IS NOT NULL THEN 
+            E'\nPARTITION BY ' || (SELECT partition_by FROM table_info)
+        ELSE '' END || ';' ||
+        CASE 
+            WHEN (SELECT ddl FROM partitions_ddl) IS NOT NULL THEN 
+                E'\n' || E'\n' || (SELECT ddl FROM partitions_ddl)
+            ELSE ''
+        END as ddl
+    FROM columns_info ci
+),
+final_original_ddl AS (
+    SELECT 
+        COALESCE(
+            (SELECT ddl FROM table_ddl),
+            (SELECT ddl FROM view_ddl),
+            (SELECT ddl FROM matview_ddl)
+        ) ||
+        CASE 
+            WHEN (SELECT ddl FROM indexes_ddl) IS NOT NULL THEN 
+                E'\n' || (SELECT ddl FROM indexes_ddl)
+            ELSE ''
+        END ||
+        CASE 
+            WHEN (SELECT ddl FROM table_comment_ddl) != '' THEN 
+                E'\n' || (SELECT ddl FROM table_comment_ddl)
+            ELSE ''
+        END ||
+        CASE 
+            WHEN (SELECT ddl FROM column_comments_ddl) IS NOT NULL THEN 
+                E'\n' || (SELECT ddl FROM column_comments_ddl)
+            ELSE ''
+        END as original_ddl
+)
+SELECT
+    (SELECT original_ddl FROM final_original_ddl) as ddl,
+    (SELECT ddl FROM temp_table_ddl) as temp_ddl,
+    (SELECT schema_name FROM table_info) || '.' || 
+        CASE
+            WHEN (SELECT table_name FROM table_info) ~ '^".*"$' THEN
+                '"' || regexp_replace((SELECT table_name FROM table_info), '^"|"$', '', 'g') || '_temp"'
+            ELSE
+                (SELECT table_name FROM table_info) || '_temp'
+        END as table_temp

dbhose_airflow/dq/clickhouse/empty.sql

@@ -0,0 +1 @@
+select count(*) as value, case when value = 0 then 'Fail' else 'Pass' end as result from {table}

dbhose_airflow/dq/clickhouse/future.sql

@@ -0,0 +1,34 @@
+with
+    replaceAll('{table}', '`', '') as table_name
+  , splitByChar('.', table_name) as table_parts
+select
+    have_test
+  , column_name
+  , query
+from (
+    select
+        database
+      , table
+      , toBool(countIf(type like 'Date%')) as have_test
+    from system.columns
+    where database = table_parts[1]
+        and table = table_parts[2]
+    group by 1, 2
+) as ht
+left join (
+    select 
+        name as column_name
+      , format(
+            'select count(*) as value, if(count() = 0, ''Pass'', ''Fail'') as result \n' ||
+            'from {{}} where {{}} > addMonths(today(), 1)'
+          , table_name
+          , column_name
+        ) as query
+      , database
+      , table
+    from system.columns
+    where database = table_parts[1]
+        and table = table_parts[2]
+        and type like 'Date%'
+) as qt
+using(database, table)