ingestr 0.13.0__tar.gz → 0.13.2__tar.gz

{ingestr-0.13.0 → ingestr-0.13.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.0
+Version: 0.13.2
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.0 → ingestr-0.13.2}/docs/supported-sources/clickhouse.md

@@ -7,14 +7,15 @@ Ingestr supports ClickHouse as a destination.
 The URI format for ClickHouse as a destination is as follows:
 
 ```plaintext
-clickhouse://<username>:<password>@<host>:<port>?http_port=<http_port>
+clickhouse://<username>:<password>@<host>:<port>?http_port=<http_port>&secure=<secure>
 ```
 ## URI parameters:
 - `username` (required): The username is required to authenticate with the ClickHouse server.
 - `password` (required): The password is required to authenticate the provided username.
 - `host` (required): The hostname or IP address of the ClickHouse server where the database is hosted.
 - `port` (required): The TCP port number used by the ClickHouse server.
-- `http_port` (optional): The port number to use when connecting to the ClickHouse server's HTTP interface. Make sure your ClickHouse server is configured to accept HTTP connections on the port specified by http_port. By default, ClickHouse uses port 8123.
+- `http_port` (optional): The port number to use when connecting to the ClickHouse server's HTTP interface.By default, it is set to port `8443`.
+- `secure` (optional): Set to `1` for a secure HTTPS connection or `0` for a non-secure HTTP connection. By default, it is set to `1`.
 
 ClickHouse requires a `username`, `password`, `host` and `port` to connect to the ClickHouse server. For more information, read [here](https://dlthub.com/docs/dlt-ecosystem/destinations/clickhouse#2-setup-clickhouse-database). Once you've completed the guide, you should have all the above-mentioned credentials.
 
@@ -29,3 +30,9 @@ ingestr ingest \
 This is a sample command that will copy the data from the Stripe source into Athena.
 
 <img alt="clickhouse_img" src="../media/clickhouse_img.png" />
+
+<!-- 
+    see https://github.com/dlt-hub/dlt/issues/2248
+-->
+> [!WARNING]
+> Clickhouse currently doesn't support `delete+insert`, `merge` or `scd2` incremental strategies.

{ingestr-0.13.0 → ingestr-0.13.2}/ingestr/src/destinations.py

@@ -1,6 +1,5 @@
 import base64
 import csv
-import gzip
 import json
 import os
 import shutil
@@ -8,6 +7,7 @@ import tempfile
 from urllib.parse import parse_qs, quote, urlparse
 
 import dlt
+import pyarrow.parquet  # type: ignore
 from dlt.common.configuration.specs import AwsCredentials
 from dlt.destinations.impl.clickhouse.configuration import (
     ClickHouseCredentials,
@@ -184,19 +184,17 @@ class CsvDestination(GenericSqlDestination):
         if output_path.count("/") > 1:
             os.makedirs(os.path.dirname(output_path), exist_ok=True)
 
-        with gzip.open(first_file_path, "rt", encoding="utf-8") as jsonl_file:  # type: ignore
-            with open(output_path, "w", newline="") as csv_file:
-                csv_writer = None
-                for line in jsonl_file:
-                    json_obj = filter_keys(json.loads(line))
-                    if csv_writer is None:
-                        csv_writer = csv.DictWriter(
-                            csv_file, fieldnames=json_obj.keys()
-                        )
-                        csv_writer.writeheader()
-
-                    csv_writer.writerow(json_obj)
-
+        table = pyarrow.parquet.read_table(first_file_path)
+        rows = table.to_pylist()
+        with open(output_path, "w", newline="") as csv_file:
+            csv_writer = None
+            for row in rows:
+                row = filter_keys(row)
+                if csv_writer is None:
+                    csv_writer = csv.DictWriter(csv_file, fieldnames=row.keys())
+                    csv_writer.writeheader()
+
+                csv_writer.writerow(row)
         shutil.rmtree(self.temp_path)
 
 
@@ -299,12 +297,21 @@ class ClickhouseDestination:
             raise ValueError(
                 "The TCP port of the ClickHouse server is required to establish a connection."
             )
-
+        
         query_params = parse_qs(parsed_uri.query)
+        secure = int(query_params["secure"][0]) if "secure" in query_params else 1
+
         http_port = (
-            int(query_params["http_port"][0]) if "http_port" in query_params else 8123
+            int(query_params["http_port"][0])
+            if "http_port" in query_params
+            else 8443 if secure == 1 else 8123
         )
 
+        if secure not in (0, 1):
+            raise ValueError(
+                "Invalid value for secure. Set to `1` for a secure HTTPS connection or `0` for a non-secure HTTP connection."
+            )
+
         credentials = ClickHouseCredentials(
             {
                 "host": host,
@@ -313,10 +320,9 @@ class ClickhouseDestination:
                 "password": password,
                 "database": database,
                 "http_port": http_port,
-                "secure": 0,
+                "secure": secure,
             }
         )
-
         return dlt.destinations.clickhouse(credentials=credentials)
 
     def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:

ingestr-0.13.2/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.13.2"

ingestr-0.13.0/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.13.0"