ingestr 0.13.0__tar.gz → 0.13.1__tar.gz

{ingestr-0.13.0 → ingestr-0.13.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.0
+Version: 0.13.1
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.0 → ingestr-0.13.1}/docs/supported-sources/clickhouse.md

@@ -29,3 +29,9 @@ ingestr ingest \
 This is a sample command that will copy the data from the Stripe source into Athena.
 
 <img alt="clickhouse_img" src="../media/clickhouse_img.png" />
+
+<!-- 
+    see https://github.com/dlt-hub/dlt/issues/2248
+-->
+> [!WARNING]
+> Clickhouse currently doesn't support `delete+insert`, `merge` or `scd2` incremental strategies.

{ingestr-0.13.0 → ingestr-0.13.1}/ingestr/src/destinations.py

@@ -1,6 +1,5 @@
 import base64
 import csv
-import gzip
 import json
 import os
 import shutil
@@ -8,6 +7,7 @@ import tempfile
 from urllib.parse import parse_qs, quote, urlparse
 
 import dlt
+import pyarrow.parquet  # type: ignore
 from dlt.common.configuration.specs import AwsCredentials
 from dlt.destinations.impl.clickhouse.configuration import (
     ClickHouseCredentials,
@@ -184,19 +184,17 @@ class CsvDestination(GenericSqlDestination):
         if output_path.count("/") > 1:
             os.makedirs(os.path.dirname(output_path), exist_ok=True)
 
-        with gzip.open(first_file_path, "rt", encoding="utf-8") as jsonl_file:  # type: ignore
-            with open(output_path, "w", newline="") as csv_file:
-                csv_writer = None
-                for line in jsonl_file:
-                    json_obj = filter_keys(json.loads(line))
-                    if csv_writer is None:
-                        csv_writer = csv.DictWriter(
-                            csv_file, fieldnames=json_obj.keys()
-                        )
-                        csv_writer.writeheader()
-
-                    csv_writer.writerow(json_obj)
-
+        table = pyarrow.parquet.read_table(first_file_path)
+        rows = table.to_pylist()
+        with open(output_path, "w", newline="") as csv_file:
+            csv_writer = None
+            for row in rows:
+                row = filter_keys(row)
+                if csv_writer is None:
+                    csv_writer = csv.DictWriter(csv_file, fieldnames=row.keys())
+                    csv_writer.writeheader()
+
+                csv_writer.writerow(row)
         shutil.rmtree(self.temp_path)
 
 

ingestr-0.13.1/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.13.1"

ingestr-0.13.0/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.13.0"