ingestr 0.13.0__py3-none-any.whl → 0.13.2__py3-none-any.whl

ingestr/src/destinations.py

@@ -1,6 +1,5 @@
 import base64
 import csv
-import gzip
 import json
 import os
 import shutil
@@ -8,6 +7,7 @@ import tempfile
 from urllib.parse import parse_qs, quote, urlparse
 
 import dlt
+import pyarrow.parquet  # type: ignore
 from dlt.common.configuration.specs import AwsCredentials
 from dlt.destinations.impl.clickhouse.configuration import (
     ClickHouseCredentials,
@@ -184,19 +184,17 @@ class CsvDestination(GenericSqlDestination):
         if output_path.count("/") > 1:
             os.makedirs(os.path.dirname(output_path), exist_ok=True)
 
-        with gzip.open(first_file_path, "rt", encoding="utf-8") as jsonl_file:  # type: ignore
-            with open(output_path, "w", newline="") as csv_file:
-                csv_writer = None
-                for line in jsonl_file:
-                    json_obj = filter_keys(json.loads(line))
-                    if csv_writer is None:
-                        csv_writer = csv.DictWriter(
-                            csv_file, fieldnames=json_obj.keys()
-                        )
-                        csv_writer.writeheader()
-
-                    csv_writer.writerow(json_obj)
-
+        table = pyarrow.parquet.read_table(first_file_path)
+        rows = table.to_pylist()
+        with open(output_path, "w", newline="") as csv_file:
+            csv_writer = None
+            for row in rows:
+                row = filter_keys(row)
+                if csv_writer is None:
+                    csv_writer = csv.DictWriter(csv_file, fieldnames=row.keys())
+                    csv_writer.writeheader()
+
+                csv_writer.writerow(row)
         shutil.rmtree(self.temp_path)
 
 
@@ -299,12 +297,21 @@ class ClickhouseDestination:
             raise ValueError(
                 "The TCP port of the ClickHouse server is required to establish a connection."
             )
-
+        
         query_params = parse_qs(parsed_uri.query)
+        secure = int(query_params["secure"][0]) if "secure" in query_params else 1
+
         http_port = (
-            int(query_params["http_port"][0]) if "http_port" in query_params else 8123
+            int(query_params["http_port"][0])
+            if "http_port" in query_params
+            else 8443 if secure == 1 else 8123
         )
 
+        if secure not in (0, 1):
+            raise ValueError(
+                "Invalid value for secure. Set to `1` for a secure HTTPS connection or `0` for a non-secure HTTP connection."
+            )
+
         credentials = ClickHouseCredentials(
             {
                 "host": host,
@@ -313,10 +320,9 @@ class ClickhouseDestination:
                 "password": password,
                 "database": database,
                 "http_port": http_port,
-                "secure": 0,
+                "secure": secure,
             }
         )
-
         return dlt.destinations.clickhouse(credentials=credentials)
 
     def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.13.0"
+__version__ = "0.13.2"

{ingestr-0.13.0.dist-info → ingestr-0.13.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.0
+Version: 0.13.2
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.0.dist-info → ingestr-0.13.2.dist-info}/RECORD

@@ -1,14 +1,14 @@
 ingestr/main.py,sha256=ufn8AcM2ID80ChUApJzYDjnQaurMXOkYfTm6GzAggSQ,24746
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=XDk_XqmU_He4sQ1brY3ceoZgpq_ZBZihz1gHW9MzqUk,1381
-ingestr/src/destinations.py,sha256=Lz7exgnfO1xW2l9Szkc5V7Pw6CcVQfIejkPCd0GSm2s,10926
+ingestr/src/destinations.py,sha256=aMRlgsq5ANnpSQmGqqWY8diB8DsF_WHbb667GnJ56js,11178
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
 ingestr/src/factory.py,sha256=3XM2rilA69vkkOCHNzUt1XqCOc3gLMnOnlQmW5d1V5s,4870
 ingestr/src/filters.py,sha256=0JQXeAr2APFMnW2sd-6BlAMWv93bXV17j8b5MM8sHmM,580
 ingestr/src/sources.py,sha256=VBuD6ngMHKaCLeYZ9Oe9tw67578hPc1dP_5iBNtEJdM,61683
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
-ingestr/src/version.py,sha256=DgpLNbv0e1LIEOOe54Db8_390i9pelMEFEnsBsNmyhA,23
+ingestr/src/version.py,sha256=blu6md2c3Nnj5gDBi8U36sYO3k8HcND8s7UoQBjfn3g,23
 ingestr/src/adjust/__init__.py,sha256=ULjtJqrNS6XDvUyGl0tjl12-tLyXlCgeFe2icTbtu3Q,3255
 ingestr/src/adjust/adjust_helpers.py,sha256=av97NPSn-hQtTbAC0vUSCAWYePmOiG5R-DGdMssm7FQ,3646
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
@@ -100,8 +100,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.0.dist-info/METADATA,sha256=o95nmTeSSzxMzMs42F9fz9WN-MuJ8DE4QHGfL6fQlHU,8252
-ingestr-0.13.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.0.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.0.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.0.dist-info/RECORD,,
+ingestr-0.13.2.dist-info/METADATA,sha256=ivVRv68P1AR_inmOV4_yMW8tfTnTtE7EBnA-bKDiIL4,8252
+ingestr-0.13.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.2.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.2.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.2.dist-info/RECORD,,