ingestr 0.12.11__tar.gz → 0.13.1__tar.gz

{ingestr-0.12.11 → ingestr-0.13.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.12.11
+Version: 0.13.1
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -15,6 +15,9 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Database
 Requires-Python: >=3.9
 Requires-Dist: asana==3.2.3
+Requires-Dist: clickhouse-connect==0.8.14
+Requires-Dist: clickhouse-driver==0.2.9
+Requires-Dist: clickhouse-sqlalchemy==0.2.7
 Requires-Dist: confluent-kafka>=2.6.1
 Requires-Dist: databricks-sql-connector==2.9.3
 Requires-Dist: dataclasses-json==0.6.7

{ingestr-0.12.11 → ingestr-0.13.1}/docs/.vitepress/config.mjs

@@ -59,6 +59,7 @@ export default defineConfig({
             items: [
               { text: "AWS Athena", link: "/supported-sources/athena.md" },
               { text: "AWS Redshift", link: "/supported-sources/redshift.md" },
+              { text: "ClickHouse", link: "/supported-sources/clickhouse.md" },
               { text: "Databricks", link: "/supported-sources/databricks.md" },
               { text: "DuckDB", link: "/supported-sources/duckdb.md" },
               {

ingestr-0.13.1/docs/supported-sources/clickhouse.md

@@ -0,0 +1,37 @@
+# ClickHouse
+ClickHouse is a fast, open-source, column-oriented database management system that allows for high performance data ingestion and querying.
+
+Ingestr supports ClickHouse as a destination.
+
+## URI format
+The URI format for ClickHouse as a destination is as follows:
+
+```plaintext
+clickhouse://<username>:<password>@<host>:<port>?http_port=<http_port>
+```
+## URI parameters:
+- `username` (required): The username is required to authenticate with the ClickHouse server.
+- `password` (required): The password is required to authenticate the provided username.
+- `host` (required): The hostname or IP address of the ClickHouse server where the database is hosted.
+- `port` (required): The TCP port number used by the ClickHouse server.
+- `http_port` (optional): The port number to use when connecting to the ClickHouse server's HTTP interface. Make sure your ClickHouse server is configured to accept HTTP connections on the port specified by http_port. By default, ClickHouse uses port 8123.
+
+ClickHouse requires a `username`, `password`, `host` and `port` to connect to the ClickHouse server. For more information, read [here](https://dlthub.com/docs/dlt-ecosystem/destinations/clickhouse#2-setup-clickhouse-database). Once you've completed the guide, you should have all the above-mentioned credentials.
+
+```
+ingestr ingest \
+    --source-uri "stripe://?api_key=key123" \
+    --source-table 'event' \
+    --dest-uri "clickhouse://user_123:pass123@localhost:9000" \
+    --dest-table 'stripe.event'
+```
+
+This is a sample command that will copy the data from the Stripe source into Athena.
+
+<img alt="clickhouse_img" src="../media/clickhouse_img.png" />
+
+<!-- 
+    see https://github.com/dlt-hub/dlt/issues/2248
+-->
+> [!WARNING]
+> Clickhouse currently doesn't support `delete+insert`, `merge` or `scd2` incremental strategies.

{ingestr-0.12.11 → ingestr-0.13.1}/ingestr/main.py

@@ -451,7 +451,7 @@ def ingest(
             pipelines_dir = tempfile.mkdtemp()
             is_pipelines_dir_temp = True
 
-        dlt_dest = destination.dlt_dest(uri=dest_uri)
+        dlt_dest = destination.dlt_dest(uri=dest_uri, dest_table=dest_table)
         validate_loader_file_format(dlt_dest, loader_file_format)
 
         if partition_by:

{ingestr-0.12.11 → ingestr-0.13.1}/ingestr/src/destinations.py

@@ -1,6 +1,5 @@
 import base64
 import csv
-import gzip
 import json
 import os
 import shutil
@@ -8,7 +7,11 @@ import tempfile
 from urllib.parse import parse_qs, quote, urlparse
 
 import dlt
+import pyarrow.parquet  # type: ignore
 from dlt.common.configuration.specs import AwsCredentials
+from dlt.destinations.impl.clickhouse.configuration import (
+    ClickHouseCredentials,
+)
 
 
 class GenericSqlDestination:
@@ -181,19 +184,17 @@ class CsvDestination(GenericSqlDestination):
         if output_path.count("/") > 1:
             os.makedirs(os.path.dirname(output_path), exist_ok=True)
 
-        with gzip.open(first_file_path, "rt", encoding="utf-8") as jsonl_file:  # type: ignore
-            with open(output_path, "w", newline="") as csv_file:
-                csv_writer = None
-                for line in jsonl_file:
-                    json_obj = filter_keys(json.loads(line))
-                    if csv_writer is None:
-                        csv_writer = csv.DictWriter(
-                            csv_file, fieldnames=json_obj.keys()
-                        )
-                        csv_writer.writeheader()
-
-                    csv_writer.writerow(json_obj)
-
+        table = pyarrow.parquet.read_table(first_file_path)
+        rows = table.to_pylist()
+        with open(output_path, "w", newline="") as csv_file:
+            csv_writer = None
+            for row in rows:
+                row = filter_keys(row)
+                if csv_writer is None:
+                    csv_writer = csv.DictWriter(csv_file, fieldnames=row.keys())
+                    csv_writer.writeheader()
+
+                csv_writer.writerow(row)
         shutil.rmtree(self.temp_path)
 
 
@@ -261,3 +262,68 @@ class AthenaDestination:
 
     def post_load(self):
         pass
+
+
+class ClickhouseDestination:
+    def dlt_dest(self, uri: str, **kwargs):
+        parsed_uri = urlparse(uri)
+
+        if "dest_table" in kwargs:
+            table = kwargs["dest_table"]
+            database = table.split(".")[0]
+        else:
+            database = parsed_uri.path.lstrip("/")
+
+        username = parsed_uri.username
+        if not username:
+            raise ValueError(
+                "A username is required to connect to the ClickHouse database."
+            )
+
+        password = parsed_uri.password
+        if not password:
+            raise ValueError(
+                "A password is required to authenticate with the ClickHouse database."
+            )
+
+        host = parsed_uri.hostname
+        if not host:
+            raise ValueError(
+                "The hostname or IP address of the ClickHouse server is required to establish a connection."
+            )
+
+        port = parsed_uri.port
+        if not port:
+            raise ValueError(
+                "The TCP port of the ClickHouse server is required to establish a connection."
+            )
+
+        query_params = parse_qs(parsed_uri.query)
+        http_port = (
+            int(query_params["http_port"][0]) if "http_port" in query_params else 8123
+        )
+
+        credentials = ClickHouseCredentials(
+            {
+                "host": host,
+                "port": port,
+                "username": username,
+                "password": password,
+                "database": database,
+                "http_port": http_port,
+                "secure": 0,
+            }
+        )
+
+        return dlt.destinations.clickhouse(credentials=credentials)
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:
+        table_fields = table.split(".")
+        if len(table_fields) != 2:
+            raise ValueError("Table name must be in the format <schema>.<table>")
+        return {
+            "table_name": table_fields[-1],
+        }
+
+    def post_load(self):
+        pass

{ingestr-0.12.11 → ingestr-0.13.1}/ingestr/src/factory.py

@@ -6,6 +6,7 @@ from dlt.common.destination import Destination
 from ingestr.src.destinations import (
     AthenaDestination,
     BigQueryDestination,
+    ClickhouseDestination,
     CsvDestination,
     DatabricksDestination,
     DuckDBDestination,
@@ -146,6 +147,8 @@ class SourceDestinationFactory:
         "synapse": SynapseDestination,
         "csv": CsvDestination,
         "athena": AthenaDestination,
+        "clickhouse+native": ClickhouseDestination,
+        "clickhouse": ClickhouseDestination,
     }
 
     def __init__(self, source_uri: str, destination_uri: str):

{ingestr-0.12.11 → ingestr-0.13.1}/ingestr/src/sources.py

@@ -1568,6 +1568,7 @@ class GCSSource:
 
         return readers(bucket_url, fs, path_to_file).with_resources(endpoint)
 
+
 class GoogleAdsSource:
     def handles_incrementality(self) -> bool:
         return True

ingestr-0.13.1/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.13.1"