dbhose-airflow 0.0.2.0__py3-none-any.whl → 0.0.2.1__py3-none-any.whl

dbhose_airflow/__init__.py

@@ -14,6 +14,7 @@ from pandas import DataFrame as PDFrame
 from polars import DataFrame as PLFrame
 
 from .airflow_connect import dbhose_dumper
+from .chunk_query import query_part
 from .dq_check import DQCheck
 from .move_method import MoveMethod
 
@@ -31,7 +32,7 @@ __all__ = (
     "dbhose_dumper",
 )
 __author__ = "0xMihalich"
-__version__ = "0.0.2.0"
+__version__ = "0.0.2.1"
 
 
 root_path = dirname(__file__)
@@ -104,7 +105,7 @@ class DBHose:
         self.connection_dest = connection_dest
         self.connection_src = connection_src
         self.dq_skip_check = dq_skip_check
-        self.filter_by = filter_by
+        self.filter_by = ", ".join(filter_by)
         self.drop_temp_table = drop_temp_table
         self.move_method = move_method
         self.custom_move = custom_move
@@ -333,12 +334,23 @@ class DBHose:
                 self.logger.error(wrap_frame(error_msg))
                 raise ValueError(error_msg)
 
-            self.dumper_dest.cursor.execute(self.custom_move)
+            for query in query_part(self.custom_move):
+                self.dumper_dest.cursor.execute(query)
 
             if self.dumper_dest.__class__ is not NativeDumper:
                 self.dumper_dest.connect.commit()
 
         elif self.move_method.have_sql:
+
+            if (
+                self.move_method is MoveMethod.delete
+                and self.dumper_dest.__class__ is NativeDumper
+                and len(self.filter_by.split(", ")) > 4
+            ):
+                error_msg = "Too many columns in filter_by (> 4)"
+                self.logger.error(wrap_frame(error_msg))
+                raise ValueError(error_msg)
+
             move_query = read_text(
                 mv_path.format(self.dumper_dest.dbname, self.move_method.name)
             )
@@ -357,12 +369,24 @@ class DBHose:
                 self.logger.error(wrap_frame(error_msg))
                 raise ValueError(error_msg)
 
-            self.dumper_dest.cursor.execute(move_query)
+            for query in query_part(move_query):
+                self.dumper_dest.cursor.execute(query)
 
             if self.dumper_dest.__class__ is not NativeDumper:
                 self.dumper_dest.connect.commit()
 
         else:
+            if self.move_method is MoveMethod.rewrite:
+                self.logger.info("Clear table operation start")
+                self.dumper_dest.cursor.execute(
+                    f"truncate table {self.table_dest}"
+                )
+
+                if self.dumper_dest.__class__ is not NativeDumper:
+                    self.dumper_dest.connect.commit()
+
+                self.logger.info("Clear table operation done")
+
             self.dumper_dest.write_between(self.table_dest, self.table_temp)
 
         self.logger.info(wrap_frame(f"Data moved into {self.table_dest}"))

dbhose_airflow/chunk_query.py

@@ -0,0 +1,14 @@
+from re import split
+
+
+pattern = r";(?=(?:[^']*'[^']*')*[^']*$)"
+
+
+def query_part(query: str) -> tuple[str]:
+    """Chunk multiquery to parts."""
+
+    return (
+        part.strip(";").strip()
+        for part in split(pattern, query)
+        if part.strip(";").strip()
+    )

dbhose_airflow/move_method.py

@@ -18,3 +18,4 @@ class MoveMethod(MoveType, Enum):
     custom = MoveType("custom", False, False, True)
     delete = MoveType("delete", True, True, False)
     replace = MoveType("replace", True, False, False)
+    rewrite = MoveType("rewrite", False, False, False)

{dbhose_airflow-0.0.2.0.dist-info → dbhose_airflow-0.0.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dbhose_airflow
-Version: 0.0.2.0
+Version: 0.0.2.1
 Summary: airflow class for exchanging data between DBMSs in native binary formats.
 Home-page: https://github.com/0xMihalich/dbhose_airflow
 Author: 0xMihalich
@@ -9,8 +9,8 @@ Description-Content-Type: text/markdown
 License-File: README.md
 License-File: CHANGELOG.md
 Requires-Dist: apache-airflow>=2.4.3
-Requires-Dist: native-dumper==0.3.3.0
-Requires-Dist: pgpack-dumper==0.3.3.0
+Requires-Dist: native-dumper==0.3.3.1
+Requires-Dist: pgpack-dumper==0.3.3.1
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

dbhose_airflow-0.0.2.1.dist-info/RECORD

@@ -0,0 +1,12 @@
+dbhose_airflow/__init__.py,sha256=4ezXu1GfNIk5Lzq2LR6EMe-vCeujt8W8kU8HJt2TeB8,15298
+dbhose_airflow/airflow_connect.py,sha256=unsRItnK4Q_ieMiGKEsCw8Q_8wkaXdVOfaSWLNRyujM,906
+dbhose_airflow/chunk_query.py,sha256=qtR6FM0SAEHzm08o6AzMZepyzJ3J8qd_itdFY0YJQRg,275
+dbhose_airflow/dq_check.py,sha256=VoAw8qieA5LM1a7jaMPO3AQ7QXe_-ThZ8Gy868ozjHw,689
+dbhose_airflow/dumper.py,sha256=9BEJ36yUJ9gH5PiVirLXymSKPOgABtp7Ee8U6MtEckY,1843
+dbhose_airflow/move_method.py,sha256=EkrDy2VCbL78zfZZhwWH0gF4Ijno20FP1mRfjiABrkk,532
+dbhose_airflow-0.0.2.1.dist-info/licenses/CHANGELOG.md,sha256=sQAbtKsJ8SwQCgbUoXHbb9P8Yl8-UocOhG0K8cMd70w,852
+dbhose_airflow-0.0.2.1.dist-info/licenses/README.md,sha256=-TsSFVS-bdRMNM-xhtqiZUXyD6D_lb6Uiz8LKEPGlP0,8822
+dbhose_airflow-0.0.2.1.dist-info/METADATA,sha256=erGKDeYmuZp3-Erz2e47c07AMg0FxFUwD6nAipugC3U,9434
+dbhose_airflow-0.0.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dbhose_airflow-0.0.2.1.dist-info/top_level.txt,sha256=VlTXT0CLGGcVhbG9QPw2_a8H5UV03QMjvZ-NrPy6_jM,15
+dbhose_airflow-0.0.2.1.dist-info/RECORD,,

{dbhose_airflow-0.0.2.0.dist-info → dbhose_airflow-0.0.2.1.dist-info}/licenses/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Version History
 
+## 0.0.2.1
+
+* Add MoveMethod.rewrite for full rewrite table with new data
+* Add query_part function
+* Change filter_by initialization list to string
+* Fix Clickhouse MoveMethod.delete
+* Improve execute custom query & MoveMethod operations
+* Update depends native-dumper==0.3.3.1
+* Update depends pgpack-dumper==0.3.3.1
+
 ## 0.0.2.0
 
 * Update depends native-dumper==0.3.3.0

dbhose_airflow-0.0.2.0.dist-info/RECORD

@@ -1,11 +0,0 @@
-dbhose_airflow/__init__.py,sha256=yMh49tSwt9x66RNpZYJYBLw_fXMzwUzbycyzUyp3W1g,14352
-dbhose_airflow/airflow_connect.py,sha256=unsRItnK4Q_ieMiGKEsCw8Q_8wkaXdVOfaSWLNRyujM,906
-dbhose_airflow/dq_check.py,sha256=VoAw8qieA5LM1a7jaMPO3AQ7QXe_-ThZ8Gy868ozjHw,689
-dbhose_airflow/dumper.py,sha256=9BEJ36yUJ9gH5PiVirLXymSKPOgABtp7Ee8U6MtEckY,1843
-dbhose_airflow/move_method.py,sha256=c4g7wuiwDKudrKSWP4ov1atJGIFknHCgPnY9FMf9Ymc,477
-dbhose_airflow-0.0.2.0.dist-info/licenses/CHANGELOG.md,sha256=7HHfS-x8gX3UXBOGEuPsgmlEOFWNRILjLlE3qrNaaqQ,532
-dbhose_airflow-0.0.2.0.dist-info/licenses/README.md,sha256=-TsSFVS-bdRMNM-xhtqiZUXyD6D_lb6Uiz8LKEPGlP0,8822
-dbhose_airflow-0.0.2.0.dist-info/METADATA,sha256=I0KTrMK0VYlrg9IMbjykKeo02GU-c1ZDxZKEjf4lLas,9434
-dbhose_airflow-0.0.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dbhose_airflow-0.0.2.0.dist-info/top_level.txt,sha256=VlTXT0CLGGcVhbG9QPw2_a8H5UV03QMjvZ-NrPy6_jM,15
-dbhose_airflow-0.0.2.0.dist-info/RECORD,,