ingestr 0.12.11__py3-none-any.whl → 0.13.1__py3-none-any.whl

ingestr/main.py

@@ -451,7 +451,7 @@ def ingest(
             pipelines_dir = tempfile.mkdtemp()
             is_pipelines_dir_temp = True
 
-        dlt_dest = destination.dlt_dest(uri=dest_uri)
+        dlt_dest = destination.dlt_dest(uri=dest_uri, dest_table=dest_table)
         validate_loader_file_format(dlt_dest, loader_file_format)
 
         if partition_by:

ingestr/src/destinations.py

@@ -1,6 +1,5 @@
 import base64
 import csv
-import gzip
 import json
 import os
 import shutil
@@ -8,7 +7,11 @@ import tempfile
 from urllib.parse import parse_qs, quote, urlparse
 
 import dlt
+import pyarrow.parquet  # type: ignore
 from dlt.common.configuration.specs import AwsCredentials
+from dlt.destinations.impl.clickhouse.configuration import (
+    ClickHouseCredentials,
+)
 
 
 class GenericSqlDestination:
@@ -181,19 +184,17 @@ class CsvDestination(GenericSqlDestination):
         if output_path.count("/") > 1:
             os.makedirs(os.path.dirname(output_path), exist_ok=True)
 
-        with gzip.open(first_file_path, "rt", encoding="utf-8") as jsonl_file:  # type: ignore
-            with open(output_path, "w", newline="") as csv_file:
-                csv_writer = None
-                for line in jsonl_file:
-                    json_obj = filter_keys(json.loads(line))
-                    if csv_writer is None:
-                        csv_writer = csv.DictWriter(
-                            csv_file, fieldnames=json_obj.keys()
-                        )
-                        csv_writer.writeheader()
-
-                    csv_writer.writerow(json_obj)
-
+        table = pyarrow.parquet.read_table(first_file_path)
+        rows = table.to_pylist()
+        with open(output_path, "w", newline="") as csv_file:
+            csv_writer = None
+            for row in rows:
+                row = filter_keys(row)
+                if csv_writer is None:
+                    csv_writer = csv.DictWriter(csv_file, fieldnames=row.keys())
+                    csv_writer.writeheader()
+
+                csv_writer.writerow(row)
         shutil.rmtree(self.temp_path)
 
 
@@ -261,3 +262,68 @@ class AthenaDestination:
 
     def post_load(self):
         pass
+
+
+class ClickhouseDestination:
+    def dlt_dest(self, uri: str, **kwargs):
+        parsed_uri = urlparse(uri)
+
+        if "dest_table" in kwargs:
+            table = kwargs["dest_table"]
+            database = table.split(".")[0]
+        else:
+            database = parsed_uri.path.lstrip("/")
+
+        username = parsed_uri.username
+        if not username:
+            raise ValueError(
+                "A username is required to connect to the ClickHouse database."
+            )
+
+        password = parsed_uri.password
+        if not password:
+            raise ValueError(
+                "A password is required to authenticate with the ClickHouse database."
+            )
+
+        host = parsed_uri.hostname
+        if not host:
+            raise ValueError(
+                "The hostname or IP address of the ClickHouse server is required to establish a connection."
+            )
+
+        port = parsed_uri.port
+        if not port:
+            raise ValueError(
+                "The TCP port of the ClickHouse server is required to establish a connection."
+            )
+
+        query_params = parse_qs(parsed_uri.query)
+        http_port = (
+            int(query_params["http_port"][0]) if "http_port" in query_params else 8123
+        )
+
+        credentials = ClickHouseCredentials(
+            {
+                "host": host,
+                "port": port,
+                "username": username,
+                "password": password,
+                "database": database,
+                "http_port": http_port,
+                "secure": 0,
+            }
+        )
+
+        return dlt.destinations.clickhouse(credentials=credentials)
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:
+        table_fields = table.split(".")
+        if len(table_fields) != 2:
+            raise ValueError("Table name must be in the format <schema>.<table>")
+        return {
+            "table_name": table_fields[-1],
+        }
+
+    def post_load(self):
+        pass

ingestr/src/factory.py

@@ -6,6 +6,7 @@ from dlt.common.destination import Destination
 from ingestr.src.destinations import (
     AthenaDestination,
     BigQueryDestination,
+    ClickhouseDestination,
     CsvDestination,
     DatabricksDestination,
     DuckDBDestination,
@@ -146,6 +147,8 @@ class SourceDestinationFactory:
         "synapse": SynapseDestination,
         "csv": CsvDestination,
         "athena": AthenaDestination,
+        "clickhouse+native": ClickhouseDestination,
+        "clickhouse": ClickhouseDestination,
     }
 
     def __init__(self, source_uri: str, destination_uri: str):

ingestr/src/sources.py

@@ -1568,6 +1568,7 @@ class GCSSource:
 
         return readers(bucket_url, fs, path_to_file).with_resources(endpoint)
 
+
 class GoogleAdsSource:
     def handles_incrementality(self) -> bool:
         return True

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.12.11"
+__version__ = "0.13.1"

{ingestr-0.12.11.dist-info → ingestr-0.13.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.12.11
+Version: 0.13.1
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -15,6 +15,9 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Database
 Requires-Python: >=3.9
 Requires-Dist: asana==3.2.3
+Requires-Dist: clickhouse-connect==0.8.14
+Requires-Dist: clickhouse-driver==0.2.9
+Requires-Dist: clickhouse-sqlalchemy==0.2.7
 Requires-Dist: confluent-kafka>=2.6.1
 Requires-Dist: databricks-sql-connector==2.9.3
 Requires-Dist: dataclasses-json==0.6.7

{ingestr-0.12.11.dist-info → ingestr-0.13.1.dist-info}/RECORD

@@ -1,14 +1,14 @@
-ingestr/main.py,sha256=fRWnyoPzMvvxTa61EIAP_dsKu0B_0yOwoyt0Slq9WQU,24723
+ingestr/main.py,sha256=ufn8AcM2ID80ChUApJzYDjnQaurMXOkYfTm6GzAggSQ,24746
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=XDk_XqmU_He4sQ1brY3ceoZgpq_ZBZihz1gHW9MzqUk,1381
-ingestr/src/destinations.py,sha256=zcHJIIHAZmcD9sJomd6G1Bc-1KsxnBD2aByOSV_9L3g,8850
+ingestr/src/destinations.py,sha256=tY9-1mOyGkXl5dbSiE-eOsK-jnbYIu4EaoqhKDZ5JuU,10829
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=D__Oy029z6y2OsAUMGab5K5ZmYhRXxDbD_SDc21b9Eo,4746
+ingestr/src/factory.py,sha256=3XM2rilA69vkkOCHNzUt1XqCOc3gLMnOnlQmW5d1V5s,4870
 ingestr/src/filters.py,sha256=0JQXeAr2APFMnW2sd-6BlAMWv93bXV17j8b5MM8sHmM,580
-ingestr/src/sources.py,sha256=jIq1qVj8_uOVbdrVuvs2uHkrLydd1i8XHMx5vhPVqAo,61682
+ingestr/src/sources.py,sha256=VBuD6ngMHKaCLeYZ9Oe9tw67578hPc1dP_5iBNtEJdM,61683
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
-ingestr/src/version.py,sha256=92OWM_xUUgc7wxFngCUAzVKFahsSWsF4UXOgDEn2uVI,24
+ingestr/src/version.py,sha256=Zg3oo58_HXe_ieb_PwWnYkKGH2zTvu6G2jly-7GnPGo,23
 ingestr/src/adjust/__init__.py,sha256=ULjtJqrNS6XDvUyGl0tjl12-tLyXlCgeFe2icTbtu3Q,3255
 ingestr/src/adjust/adjust_helpers.py,sha256=av97NPSn-hQtTbAC0vUSCAWYePmOiG5R-DGdMssm7FQ,3646
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
@@ -100,8 +100,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.12.11.dist-info/METADATA,sha256=fxNa7pb3GLEvLuUjHSOviflBwIBJto0ck1PyQp893jU,8127
-ingestr-0.12.11.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.12.11.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.12.11.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.12.11.dist-info/RECORD,,
+ingestr-0.13.1.dist-info/METADATA,sha256=c2bUEbUHRYoKiHXNgHSbzNsQL3M2dLiCZK1fzbjzylU,8252
+ingestr-0.13.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.1.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.1.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.1.dist-info/RECORD,,