dbhose-airflow 0.0.2.0__py3-none-any.whl → 0.0.2.2__py3-none-any.whl

dbhose_airflow/move/clickhouse/delete.sql

@@ -0,0 +1,81 @@
+with
+    splitByChar(',', replaceAll('{filter_by}', ' ', '')) as columns,
+    length(columns) as columns_count,
+    (
+        select groupArray(tuple({filter_by}))
+        from (
+            select distinct {filter_by}
+            from {table_temp}
+        )
+    ) as values_tuples
+select
+    true as is_avaliable,
+    'ALTER TABLE {table_dest} DELETE WHERE ' ||
+    if(columns_count > 1, '(' || arrayStringConcat(columns, ', ') || ')', arrayStringConcat(columns, ', ')) ||
+    ' IN (' ||
+    arrayStringConcat(
+        arrayDistinct(
+            arrayMap(
+                tuple ->
+                    if(columns_count = 1,
+                        (case
+                            when isNull(tuple.1) then 'NULL'
+                            when toTypeName(tuple.1) REGEXP 'U?Int|B?Float|Decimal|Bool' then toString(tuple.1)
+                            else '\'' || replaceAll(toString(tuple.1), '\'', '\\\'') || '\''
+                        end),
+                    if(columns_count = 2,
+                        '(' ||
+                        (case
+                            when isNull(tuple.1) then 'NULL'
+                            when toTypeName(tuple.1) REGEXP 'U?Int|B?Float|Decimal|Bool' then toString(tuple.1)
+                            else '\'' || replaceAll(toString(tuple.1), '\'', '\\\'') || '\''
+                        end) || ', ' ||
+                        (case
+                            when isNull(tuple.2) then 'NULL'
+                            when toTypeName(tuple.2) REGEXP 'U?Int|B?Float|Decimal|Bool' then toString(tuple.2)
+                            else '\'' || replaceAll(toString(tuple.2), '\'', '\\\'') || '\''
+                        end) || ')',
+                    if(columns_count = 3,
+                        '(' ||
+                        (case
+                            when isNull(tuple.1) then 'NULL'
+                            when toTypeName(tuple.1) REGEXP 'U?Int|B?Float|Decimal|Bool' then toString(tuple.1)
+                            else '\'' || replaceAll(toString(tuple.1), '\'', '\\\'') || '\''
+                        end) || ', ' ||
+                        (case
+                            when isNull(tuple.2) then 'NULL'
+                            when toTypeName(tuple.2) REGEXP 'U?Int|B?Float|Decimal|Bool' then toString(tuple.2)
+                            else '\'' || replaceAll(toString(tuple.2), '\'', '\\\'') || '\''
+                        end) || ', ' ||
+                        (case
+                            WHEN isNull(tuple.3) THEN 'NULL'
+                            WHEN toTypeName(tuple.3) REGEXP 'U?Int|B?Float|Decimal|Bool' THEN toString(tuple.3)
+                            ELSE '\'' || replaceAll(toString(tuple.3), '\'', '\\\'') || '\''
+                        END) || ')',
+                        '(' ||
+                        (CASE
+                            WHEN isNull(tuple.1) THEN 'NULL'
+                            WHEN toTypeName(tuple.1) REGEXP 'U?Int|B?Float|Decimal|Bool' THEN toString(tuple.1)
+                            ELSE '\'' || replaceAll(toString(tuple.1), '\'', '\\\'') || '\''
+                        END) || ', ' ||
+                        (CASE
+                            WHEN isNull(tuple.2) THEN 'NULL'
+                            WHEN toTypeName(tuple.2) REGEXP 'U?Int|B?Float|Decimal|Bool' THEN toString(tuple.2)
+                            ELSE '\'' || replaceAll(toString(tuple.2), '\'', '\\\'') || '\''
+                        END) || ', ' ||
+                        (case
+                            when isNull(tuple.3) then 'NULL'
+                            when toTypeName(tuple.3) REGEXP 'U?Int|B?Float|Decimal|Bool' then toString(tuple.3)
+                            else '\'' || replaceAll(toString(tuple.3), '\'', '\\\'') || '\''
+                        end) || ', ' ||
+                        (case
+                            when isNull(tuple.4) then 'NULL'
+                            when toTypeName(tuple.4) REGEXP 'U?Int|B?Float|Decimal|Bool' then toString(tuple.4)
+                            else '\'' || replaceAll(toString(tuple.4), '\'', '\\\'') || '\''
+                        end) || ')'
+                    ))),
+                values_tuples
+            )
+        ),
+        ', '
+    ) || ');\nINSERT INTO {table_dest}\nSELECT * FROM {table_temp};' as move_query

dbhose_airflow/move/clickhouse/replace.sql

@@ -0,0 +1,24 @@
+with 
+    '{table_temp}' as table_temp
+  , '{table_dest}' as table_dest
+  , partitions as (
+    select partition 
+    from system.parts 
+    where database = splitByChar('.', table_temp)[1]
+        and table = splitByChar('.', table_temp)[2]
+        and partition <> 'tuple()'
+        and active
+    order by partition
+)
+select
+    toBool(count() > 0) as is_avaliable
+  , if(
+        is_avaliable
+      , 'alter table ' || table_dest || ' ' ||
+        arrayStringConcat(
+            arrayMap(
+                p -> 'replace partition ' || p || ' from ' || table_temp
+              , groupArray(partition)
+            ), ', ')
+      , '') as move_query
+from partitions

dbhose_airflow/move/greenplum/delete.sql

@@ -0,0 +1,10 @@
+with filter_columns as (
+    select array_length(string_to_array('{filter_by}', ', '), 1) as col_count
+)
+select
+    true as is_avaliable,
+    'DELETE FROM {table_dest} WHERE EXISTS(' || E'\nSELECT 1 FROM {table_temp}\nWHERE ' || 
+    (select string_agg('{table_dest}.' || trim(col) || ' = {table_temp}.' || trim(col), E'\nAND ') 
+     from unnest(string_to_array('{filter_by}', ',')) as col) || E'\n);\n' ||
+    E'INSERT INTO {table_dest}\nSELECT * FROM {table_temp};' as move_query
+from filter_columns

dbhose_airflow/move/greenplum/replace.sql

@@ -0,0 +1,60 @@
+with 
+    matching_partitions as (
+        select
+            table_partition
+          , partition_constraint
+        from (
+            select
+                child.relname as table_partition
+              , pg_get_expr(child.relpartbound, child.oid) as partition_constraint
+            from pg_inherits
+            join pg_class child
+                on pg_inherits.inhrelid = child.oid
+            where pg_inherits.inhparent = '{table_dest}'::regclass
+        ) as dest_tbl
+        join (
+            select
+                pg_get_expr(child.relpartbound, child.oid) as partition_constraint
+              , child.oid as partition_oid
+            from pg_inherits
+            join pg_class child
+                on pg_inherits.inhrelid = child.oid
+            where pg_inherits.inhparent = '{table_temp}'::regclass
+        ) as temp_tbl
+        using(partition_constraint)
+        where exists (
+            select 1
+            from pg_class
+            where oid = temp_tbl.partition_oid
+                and reltuples > 0
+        )
+    ),
+    detach_commands as (
+        select string_agg(
+            'ALTER TABLE {table_dest} DETACH PARTITION ' ||
+            table_partition || ';'
+          , E'\n'
+        ) as commands
+        from matching_partitions
+    ),
+    attach_commands as (
+        select string_agg(
+            'ALTER TABLE {table_dest} ATTACH PARTITION {table_temp} ' ||
+            partition_constraint || ';'
+          , E'\n'
+        ) as commands
+        from matching_partitions
+    ),
+    drop_commands as (
+        select string_agg(
+            'DROP TABLE ' || table_partition || ';'
+          , E'\n'
+        ) as commands
+        from matching_partitions
+    )
+select
+    (select case when count(*) > 0 then true else false end
+     from matching_partitions) as is_avaliable
+  , (select commands from detach_commands) || E'\n' ||
+    (select commands from attach_commands) || E'\n' ||
+    (select commands from drop_commands) as move_query

dbhose_airflow/move/postgres/delete.sql

@@ -0,0 +1,10 @@
+with filter_columns as (
+    select array_length(string_to_array('{filter_by}', ', '), 1) as col_count
+)
+select
+    true as is_avaliable,
+    'DELETE FROM {table_dest} WHERE EXISTS(' || E'\nSELECT 1 FROM {table_temp}\nWHERE ' || 
+    (select string_agg('{table_dest}.' || trim(col) || ' = {table_temp}.' || trim(col), E'\nAND ') 
+     from unnest(string_to_array('{filter_by}', ',')) as col) || E'\n);\n' ||
+    E'INSERT INTO {table_dest}\nSELECT * FROM {table_temp};' as move_query
+from filter_columns

dbhose_airflow/move/postgres/replace.sql

@@ -0,0 +1,60 @@
+with 
+    matching_partitions as (
+        select
+            table_partition
+          , partition_constraint
+        from (
+            select
+                child.relname as table_partition
+              , pg_get_expr(child.relpartbound, child.oid) as partition_constraint
+            from pg_inherits
+            join pg_class child
+                on pg_inherits.inhrelid = child.oid
+            where pg_inherits.inhparent = '{table_dest}'::regclass
+        ) as dest_tbl
+        join (
+            select
+                pg_get_expr(child.relpartbound, child.oid) as partition_constraint
+              , child.oid as partition_oid
+            from pg_inherits
+            join pg_class child
+                on pg_inherits.inhrelid = child.oid
+            where pg_inherits.inhparent = '{table_temp}'::regclass
+        ) as temp_tbl
+        using(partition_constraint)
+        where exists (
+            select 1
+            from pg_class
+            where oid = temp_tbl.partition_oid
+                and reltuples > 0
+        )
+    ),
+    detach_commands as (
+        select string_agg(
+            'ALTER TABLE {table_dest} DETACH PARTITION ' ||
+            table_partition || ';'
+          , E'\n'
+        ) as commands
+        from matching_partitions
+    ),
+    attach_commands as (
+        select string_agg(
+            'ALTER TABLE {table_dest} ATTACH PARTITION {table_temp} ' ||
+            partition_constraint || ';'
+          , E'\n'
+        ) as commands
+        from matching_partitions
+    ),
+    drop_commands as (
+        select string_agg(
+            'DROP TABLE ' || table_partition || ';'
+          , E'\n'
+        ) as commands
+        from matching_partitions
+    )
+select
+    (select case when count(*) > 0 then true else false end
+     from matching_partitions) as is_avaliable
+  , (select commands from detach_commands) || E'\n' ||
+    (select commands from attach_commands) || E'\n' ||
+    (select commands from drop_commands) as move_query

dbhose_airflow/move_method.py

@@ -18,3 +18,4 @@ class MoveMethod(MoveType, Enum):
     custom = MoveType("custom", False, False, True)
     delete = MoveType("delete", True, True, False)
     replace = MoveType("replace", True, False, False)
+    rewrite = MoveType("rewrite", False, False, False)

{dbhose_airflow-0.0.2.0.dist-info → dbhose_airflow-0.0.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dbhose_airflow
-Version: 0.0.2.0
+Version: 0.0.2.2
 Summary: airflow class for exchanging data between DBMSs in native binary formats.
 Home-page: https://github.com/0xMihalich/dbhose_airflow
 Author: 0xMihalich
@@ -9,8 +9,8 @@ Description-Content-Type: text/markdown
 License-File: README.md
 License-File: CHANGELOG.md
 Requires-Dist: apache-airflow>=2.4.3
-Requires-Dist: native-dumper==0.3.3.0
-Requires-Dist: pgpack-dumper==0.3.3.0
+Requires-Dist: native-dumper==0.3.3.1
+Requires-Dist: pgpack-dumper==0.3.3.1
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

dbhose_airflow-0.0.2.2.dist-info/RECORD

@@ -0,0 +1,43 @@
+dbhose_airflow/LOGO,sha256=zfn071DUmSt3OzqIN-knPqoYDqUNbxO2ALfdDaEII6A,783
+dbhose_airflow/__init__.py,sha256=-vyzywoaV3-AjGNxBp97UnFZ0O7A-Qr40Cg0UMvF91c,15298
+dbhose_airflow/airflow_connect.py,sha256=unsRItnK4Q_ieMiGKEsCw8Q_8wkaXdVOfaSWLNRyujM,906
+dbhose_airflow/chunk_query.py,sha256=qtR6FM0SAEHzm08o6AzMZepyzJ3J8qd_itdFY0YJQRg,275
+dbhose_airflow/dq_check.py,sha256=VoAw8qieA5LM1a7jaMPO3AQ7QXe_-ThZ8Gy868ozjHw,689
+dbhose_airflow/dumper.py,sha256=9BEJ36yUJ9gH5PiVirLXymSKPOgABtp7Ee8U6MtEckY,1843
+dbhose_airflow/move_method.py,sha256=EkrDy2VCbL78zfZZhwWH0gF4Ijno20FP1mRfjiABrkk,532
+dbhose_airflow/ddl/clickhouse.sql,sha256=HKIB2GhWgAGlipC4vS5EUI-YcmFnG34u5nm4bTyDkkU,2617
+dbhose_airflow/ddl/greenplum.sql,sha256=F_tCfaTz8YJ4bt2XPWa3Xln4Pe7ar8SKj-eznc3gWqs,9322
+dbhose_airflow/ddl/postgres.sql,sha256=qylCEgaY0UqcBtRwan5jH8CD3MCKVtaLqThodSgteR0,8256
+dbhose_airflow/dq/clickhouse/empty.sql,sha256=noHKmdqwhOGWqU30qW1lQDShOPPtu80FXDVZ-Qjxzmo,96
+dbhose_airflow/dq/clickhouse/future.sql,sha256=_pXiaR2TsEk0ZNU911rRJe0HCOy9oSHG21kxDtZ2AxU,846
+dbhose_airflow/dq/clickhouse/infinity.sql,sha256=nrb1Pgaik7ty3egXIU49gf0N-9TCq-OvjlFCZZKHVn0,836
+dbhose_airflow/dq/clickhouse/nan.sql,sha256=oe1lUWGXqBrEfVH2JauCgAqD_NVJVAl62i8fMkpkDgE,855
+dbhose_airflow/dq/clickhouse/sum.sql,sha256=wwtLpuaszzs-bpkiB2zmAHEvlH6cWCD7HDQcHy7pZAo,1559
+dbhose_airflow/dq/clickhouse/total.sql,sha256=lhw4PLEKN2Mn7E1FXIKg4l1IZ6cAHEZkPcLCbKbeP_4,37
+dbhose_airflow/dq/clickhouse/uniq.sql,sha256=ZhyRgqBVEPxzfnpFAbAUB71Vt4OFoR22hyawF52NMDE,115
+dbhose_airflow/dq/greenplum/empty.sql,sha256=bSlS58_VtWz8F3hJfbaXZArVV9QLrFCXUJmghWB52TQ,99
+dbhose_airflow/dq/greenplum/future.sql,sha256=nN361zV_Cr8MxRk5n7_xN9-OGAgptY9BbuftacXV_uE,1590
+dbhose_airflow/dq/greenplum/infinity.sql,sha256=pWx-vlloZ6fGPWPIx1MKgwia66p9P-I_HaOUOwavYIM,1699
+dbhose_airflow/dq/greenplum/nan.sql,sha256=J96yT0LKlgCE0NfbDtpBIy4sIXw7DHBfy9Td2cs_JPo,1560
+dbhose_airflow/dq/greenplum/sum.sql,sha256=1uq6V2hXn3HJhQrAM_id0FCZd-fbbfdXUNLTMJyDBso,1642
+dbhose_airflow/dq/greenplum/total.sql,sha256=lhw4PLEKN2Mn7E1FXIKg4l1IZ6cAHEZkPcLCbKbeP_4,37
+dbhose_airflow/dq/greenplum/uniq.sql,sha256=myYh7TTc-c-ixnXwMibopz4bMr9qnkvU-29nt89cq8o,227
+dbhose_airflow/dq/postgres/empty.sql,sha256=bSlS58_VtWz8F3hJfbaXZArVV9QLrFCXUJmghWB52TQ,99
+dbhose_airflow/dq/postgres/future.sql,sha256=nN361zV_Cr8MxRk5n7_xN9-OGAgptY9BbuftacXV_uE,1590
+dbhose_airflow/dq/postgres/infinity.sql,sha256=pWx-vlloZ6fGPWPIx1MKgwia66p9P-I_HaOUOwavYIM,1699
+dbhose_airflow/dq/postgres/nan.sql,sha256=J96yT0LKlgCE0NfbDtpBIy4sIXw7DHBfy9Td2cs_JPo,1560
+dbhose_airflow/dq/postgres/sum.sql,sha256=1uq6V2hXn3HJhQrAM_id0FCZd-fbbfdXUNLTMJyDBso,1642
+dbhose_airflow/dq/postgres/total.sql,sha256=lhw4PLEKN2Mn7E1FXIKg4l1IZ6cAHEZkPcLCbKbeP_4,37
+dbhose_airflow/dq/postgres/uniq.sql,sha256=myYh7TTc-c-ixnXwMibopz4bMr9qnkvU-29nt89cq8o,227
+dbhose_airflow/move/clickhouse/delete.sql,sha256=qyQ3YrDqBx4AhRMqEvcfIJAP4UauVEys_BshuMsCgUs,4316
+dbhose_airflow/move/clickhouse/replace.sql,sha256=B44YbVK3wcCOPhAp8-_yX3yxnpTjpJF7xx_fCRRxIBE,666
+dbhose_airflow/move/greenplum/delete.sql,sha256=OJJqu3CmbipMUl_Oyw48yzRSfzchzyu4YAcu9gAscFc,502
+dbhose_airflow/move/greenplum/replace.sql,sha256=Pk9g-3SBC78bB_s8ECh_6tPDqTvL1wxKNjzrTakfrL0,1976
+dbhose_airflow/move/postgres/delete.sql,sha256=OJJqu3CmbipMUl_Oyw48yzRSfzchzyu4YAcu9gAscFc,502
+dbhose_airflow/move/postgres/replace.sql,sha256=Pk9g-3SBC78bB_s8ECh_6tPDqTvL1wxKNjzrTakfrL0,1976
+dbhose_airflow-0.0.2.2.dist-info/licenses/CHANGELOG.md,sha256=rhtwm1Kf58B5L3409BoQogKd9kEQJek7dcvJqkklx3E,892
+dbhose_airflow-0.0.2.2.dist-info/licenses/README.md,sha256=-TsSFVS-bdRMNM-xhtqiZUXyD6D_lb6Uiz8LKEPGlP0,8822
+dbhose_airflow-0.0.2.2.dist-info/METADATA,sha256=yutBySs5OmjKjK91u3WQwu7bAuDVH5zBISDdXAHywVU,9434
+dbhose_airflow-0.0.2.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dbhose_airflow-0.0.2.2.dist-info/top_level.txt,sha256=VlTXT0CLGGcVhbG9QPw2_a8H5UV03QMjvZ-NrPy6_jM,15
+dbhose_airflow-0.0.2.2.dist-info/RECORD,,

{dbhose_airflow-0.0.2.0.dist-info → dbhose_airflow-0.0.2.2.dist-info}/licenses/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Version History
 
+## 0.0.2.2
+
+* Fix include_package_data
+
+## 0.0.2.1
+
+* Add MoveMethod.rewrite for full rewrite table with new data
+* Add query_part function
+* Change filter_by initialization list to string
+* Fix Clickhouse MoveMethod.delete
+* Improve execute custom query & MoveMethod operations
+* Update depends native-dumper==0.3.3.1
+* Update depends pgpack-dumper==0.3.3.1
+
 ## 0.0.2.0
 
 * Update depends native-dumper==0.3.3.0

dbhose_airflow-0.0.2.0.dist-info/RECORD

@@ -1,11 +0,0 @@
-dbhose_airflow/__init__.py,sha256=yMh49tSwt9x66RNpZYJYBLw_fXMzwUzbycyzUyp3W1g,14352
-dbhose_airflow/airflow_connect.py,sha256=unsRItnK4Q_ieMiGKEsCw8Q_8wkaXdVOfaSWLNRyujM,906
-dbhose_airflow/dq_check.py,sha256=VoAw8qieA5LM1a7jaMPO3AQ7QXe_-ThZ8Gy868ozjHw,689
-dbhose_airflow/dumper.py,sha256=9BEJ36yUJ9gH5PiVirLXymSKPOgABtp7Ee8U6MtEckY,1843
-dbhose_airflow/move_method.py,sha256=c4g7wuiwDKudrKSWP4ov1atJGIFknHCgPnY9FMf9Ymc,477
-dbhose_airflow-0.0.2.0.dist-info/licenses/CHANGELOG.md,sha256=7HHfS-x8gX3UXBOGEuPsgmlEOFWNRILjLlE3qrNaaqQ,532
-dbhose_airflow-0.0.2.0.dist-info/licenses/README.md,sha256=-TsSFVS-bdRMNM-xhtqiZUXyD6D_lb6Uiz8LKEPGlP0,8822
-dbhose_airflow-0.0.2.0.dist-info/METADATA,sha256=I0KTrMK0VYlrg9IMbjykKeo02GU-c1ZDxZKEjf4lLas,9434
-dbhose_airflow-0.0.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dbhose_airflow-0.0.2.0.dist-info/top_level.txt,sha256=VlTXT0CLGGcVhbG9QPw2_a8H5UV03QMjvZ-NrPy6_jM,15
-dbhose_airflow-0.0.2.0.dist-info/RECORD,,