dbhose-airflow 0.0.1.0__py3-none-any.whl → 0.0.2.1__py3-none-any.whl

dbhose_airflow/__init__.py

@@ -14,6 +14,7 @@ from pandas import DataFrame as PDFrame
 from polars import DataFrame as PLFrame
 
 from .airflow_connect import dbhose_dumper
+from .chunk_query import query_part
 from .dq_check import DQCheck
 from .move_method import MoveMethod
 
@@ -31,7 +32,7 @@ __all__ = (
     "dbhose_dumper",
 )
 __author__ = "0xMihalich"
-__version__ = "0.0.1.0"
+__version__ = "0.0.2.1"
 
 
 root_path = dirname(__file__)
@@ -104,7 +105,7 @@ class DBHose:
         self.connection_dest = connection_dest
         self.connection_src = connection_src
         self.dq_skip_check = dq_skip_check
-        self.filter_by = filter_by
+        self.filter_by = ", ".join(filter_by)
         self.drop_temp_table = drop_temp_table
         self.move_method = move_method
         self.custom_move = custom_move
@@ -333,12 +334,23 @@ class DBHose:
                 self.logger.error(wrap_frame(error_msg))
                 raise ValueError(error_msg)
 
-            self.dumper_dest.cursor.execute(self.custom_move)
+            for query in query_part(self.custom_move):
+                self.dumper_dest.cursor.execute(query)
 
             if self.dumper_dest.__class__ is not NativeDumper:
                 self.dumper_dest.connect.commit()
 
         elif self.move_method.have_sql:
+
+            if (
+                self.move_method is MoveMethod.delete
+                and self.dumper_dest.__class__ is NativeDumper
+                and len(self.filter_by.split(", ")) > 4
+            ):
+                error_msg = "Too many columns in filter_by (> 4)"
+                self.logger.error(wrap_frame(error_msg))
+                raise ValueError(error_msg)
+
             move_query = read_text(
                 mv_path.format(self.dumper_dest.dbname, self.move_method.name)
             )
@@ -349,7 +361,7 @@ class DBHose:
             ))
             is_avaliable, move_query = tuple(*reader.to_rows())
 
-            if not is_avaliable:
+            if not is_avaliable or not move_query:
                 error_msg = (
                     f"Method {self.move_method.name} is not available for "
                     f"{self.table_dest}. Use another method."
@@ -357,12 +369,24 @@ class DBHose:
                 self.logger.error(wrap_frame(error_msg))
                 raise ValueError(error_msg)
 
-            self.dumper_dest.cursor.execute(move_query)
+            for query in query_part(move_query):
+                self.dumper_dest.cursor.execute(query)
 
             if self.dumper_dest.__class__ is not NativeDumper:
                 self.dumper_dest.connect.commit()
 
         else:
+            if self.move_method is MoveMethod.rewrite:
+                self.logger.info("Clear table operation start")
+                self.dumper_dest.cursor.execute(
+                    f"truncate table {self.table_dest}"
+                )
+
+                if self.dumper_dest.__class__ is not NativeDumper:
+                    self.dumper_dest.connect.commit()
+
+                self.logger.info("Clear table operation done")
+
             self.dumper_dest.write_between(self.table_dest, self.table_temp)
 
         self.logger.info(wrap_frame(f"Data moved into {self.table_dest}"))

dbhose_airflow/chunk_query.py

@@ -0,0 +1,14 @@
+from re import split
+
+
+pattern = r";(?=(?:[^']*'[^']*')*[^']*$)"
+
+
+def query_part(query: str) -> tuple[str]:
+    """Chunk multiquery to parts."""
+
+    return (
+        part.strip(";").strip()
+        for part in split(pattern, query)
+        if part.strip(";").strip()
+    )

dbhose_airflow/move_method.py

@@ -18,3 +18,4 @@ class MoveMethod(MoveType, Enum):
     custom = MoveType("custom", False, False, True)
     delete = MoveType("delete", True, True, False)
     replace = MoveType("replace", True, False, False)
+    rewrite = MoveType("rewrite", False, False, False)

{dbhose_airflow-0.0.1.0.dist-info → dbhose_airflow-0.0.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dbhose_airflow
-Version: 0.0.1.0
+Version: 0.0.2.1
 Summary: airflow class for exchanging data between DBMSs in native binary formats.
 Home-page: https://github.com/0xMihalich/dbhose_airflow
 Author: 0xMihalich
@@ -9,8 +9,8 @@ Description-Content-Type: text/markdown
 License-File: README.md
 License-File: CHANGELOG.md
 Requires-Dist: apache-airflow>=2.4.3
-Requires-Dist: native-dumper==0.3.2.3
-Requires-Dist: pgpack-dumper==0.3.2.2
+Requires-Dist: native-dumper==0.3.3.1
+Requires-Dist: pgpack-dumper==0.3.3.1
 Dynamic: author
 Dynamic: author-email
 Dynamic: description
@@ -84,7 +84,7 @@ DBHose(
 - **`move_method`** (`MoveMethod`) - метод перемещения данных (по умолчанию `MoveMethod.replace`)
 - **`custom_move`** (`str`, optional) - пользовательский SQL запрос для перемещения данных
 - **`compress_method`** (`CompressionMethod`) - метод сжатия для дампов (по умолчанию `CompressionMethod.ZSTD`)
-- **`timeout`** (`int`) - таймаут операций с БД в секундах (по умолчанию `DBMS_DEFAULT_TIMEOUT_SEC`)
+- **`timeout`** (`int`) - таймаут операций с БД в секундах (по умолчанию `DBMS_DEFAULT_TIMEOUT_SEC` = 300)
 
 ## Методы
 
@@ -135,9 +135,14 @@ DBHose(
 ## Пример использования в DAG
 
 ```python
+from datetime import datetime
+
 from airflow import DAG
 from airflow.operators.python import PythonOperator
-from datetime import datetime
+from dbhose_airflow import (
+    DBHose,
+    MoveMethod,
+)
 
 def transfer_data():
     # Перенос данных из PostgreSQL в ClickHouse

dbhose_airflow-0.0.2.1.dist-info/RECORD

@@ -0,0 +1,12 @@
+dbhose_airflow/__init__.py,sha256=4ezXu1GfNIk5Lzq2LR6EMe-vCeujt8W8kU8HJt2TeB8,15298
+dbhose_airflow/airflow_connect.py,sha256=unsRItnK4Q_ieMiGKEsCw8Q_8wkaXdVOfaSWLNRyujM,906
+dbhose_airflow/chunk_query.py,sha256=qtR6FM0SAEHzm08o6AzMZepyzJ3J8qd_itdFY0YJQRg,275
+dbhose_airflow/dq_check.py,sha256=VoAw8qieA5LM1a7jaMPO3AQ7QXe_-ThZ8Gy868ozjHw,689
+dbhose_airflow/dumper.py,sha256=9BEJ36yUJ9gH5PiVirLXymSKPOgABtp7Ee8U6MtEckY,1843
+dbhose_airflow/move_method.py,sha256=EkrDy2VCbL78zfZZhwWH0gF4Ijno20FP1mRfjiABrkk,532
+dbhose_airflow-0.0.2.1.dist-info/licenses/CHANGELOG.md,sha256=sQAbtKsJ8SwQCgbUoXHbb9P8Yl8-UocOhG0K8cMd70w,852
+dbhose_airflow-0.0.2.1.dist-info/licenses/README.md,sha256=-TsSFVS-bdRMNM-xhtqiZUXyD6D_lb6Uiz8LKEPGlP0,8822
+dbhose_airflow-0.0.2.1.dist-info/METADATA,sha256=erGKDeYmuZp3-Erz2e47c07AMg0FxFUwD6nAipugC3U,9434
+dbhose_airflow-0.0.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dbhose_airflow-0.0.2.1.dist-info/top_level.txt,sha256=VlTXT0CLGGcVhbG9QPw2_a8H5UV03QMjvZ-NrPy6_jM,15
+dbhose_airflow-0.0.2.1.dist-info/RECORD,,

dbhose_airflow-0.0.2.1.dist-info/licenses/CHANGELOG.md

@@ -0,0 +1,32 @@
+# Version History
+
+## 0.0.2.1
+
+* Add MoveMethod.rewrite for full rewrite table with new data
+* Add query_part function
+* Change filter_by initialization list to string
+* Fix Clickhouse MoveMethod.delete
+* Improve execute custom query & MoveMethod operations
+* Update depends native-dumper==0.3.3.1
+* Update depends pgpack-dumper==0.3.3.1
+
+## 0.0.2.0
+
+* Update depends native-dumper==0.3.3.0
+* Update depends pgpack-dumper==0.3.3.0
+* Update README.md
+* Add create partition into postgres and greenplum ddl queryes
+* Improve delete.sql for greenplum and postgres
+
+## 0.0.1.0
+
+* Update depends native-dumper==0.3.2.3
+* Update depends pgpack-dumper==0.3.2.2
+* Move old README.md into OLD_DOCS.md
+* Create new README.md
+* Delete dbhose-utils from depends
+* Rename repository dbhose -> dbhose_airflow
+
+## 0.0.0.1
+
+First version of the library dbhose_airflow

{dbhose_airflow-0.0.1.0.dist-info → dbhose_airflow-0.0.2.1.dist-info}/licenses/README.md

@@ -63,7 +63,7 @@ DBHose(
 - **`move_method`** (`MoveMethod`) - метод перемещения данных (по умолчанию `MoveMethod.replace`)
 - **`custom_move`** (`str`, optional) - пользовательский SQL запрос для перемещения данных
 - **`compress_method`** (`CompressionMethod`) - метод сжатия для дампов (по умолчанию `CompressionMethod.ZSTD`)
-- **`timeout`** (`int`) - таймаут операций с БД в секундах (по умолчанию `DBMS_DEFAULT_TIMEOUT_SEC`)
+- **`timeout`** (`int`) - таймаут операций с БД в секундах (по умолчанию `DBMS_DEFAULT_TIMEOUT_SEC` = 300)
 
 ## Методы
 
@@ -114,9 +114,14 @@ DBHose(
 ## Пример использования в DAG
 
 ```python
+from datetime import datetime
+
 from airflow import DAG
 from airflow.operators.python import PythonOperator
-from datetime import datetime
+from dbhose_airflow import (
+    DBHose,
+    MoveMethod,
+)
 
 def transfer_data():
     # Перенос данных из PostgreSQL в ClickHouse

dbhose_airflow-0.0.1.0.dist-info/RECORD

@@ -1,11 +0,0 @@
-dbhose_airflow/__init__.py,sha256=bDVnzUxA_AD1-rrJqpJPGN76yPI76NDrkAO1DSavMYA,14334
-dbhose_airflow/airflow_connect.py,sha256=unsRItnK4Q_ieMiGKEsCw8Q_8wkaXdVOfaSWLNRyujM,906
-dbhose_airflow/dq_check.py,sha256=VoAw8qieA5LM1a7jaMPO3AQ7QXe_-ThZ8Gy868ozjHw,689
-dbhose_airflow/dumper.py,sha256=9BEJ36yUJ9gH5PiVirLXymSKPOgABtp7Ee8U6MtEckY,1843
-dbhose_airflow/move_method.py,sha256=c4g7wuiwDKudrKSWP4ov1atJGIFknHCgPnY9FMf9Ymc,477
-dbhose_airflow-0.0.1.0.dist-info/licenses/CHANGELOG.md,sha256=ps8G7NfFWWW2CcXLPl6OcE_mVcx7dgahWK6J6Hh1xQI,309
-dbhose_airflow-0.0.1.0.dist-info/licenses/README.md,sha256=SEcBu3s27QTwMLCVlQbXfErdRWsQtfFfbCE9b6FWwoI,8756
-dbhose_airflow-0.0.1.0.dist-info/METADATA,sha256=V20eV-fvWe-aylcrsjaOVgq0CWS0ONwV_rnPnAwbhRM,9368
-dbhose_airflow-0.0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dbhose_airflow-0.0.1.0.dist-info/top_level.txt,sha256=VlTXT0CLGGcVhbG9QPw2_a8H5UV03QMjvZ-NrPy6_jM,15
-dbhose_airflow-0.0.1.0.dist-info/RECORD,,

dbhose_airflow-0.0.1.0.dist-info/licenses/CHANGELOG.md

@@ -1,14 +0,0 @@
-# Version History
-
-## 0.0.1.0
-
-* Update depends native-dumper==0.3.2.3
-* Update depends pgpack-dumper==0.3.2.2
-* Move old README.md into OLD_DOCS.md
-* Create new README.md
-* Delete dbhose-utils from depends
-* Rename repository dbhose -> dbhose_airflow
-
-## 0.0.0.1
-
-First version of the library dbhose_airflow