ingestr 0.13.3__py3-none-any.whl → 0.13.5__py3-none-any.whl

ingestr/src/destinations.py

@@ -7,12 +7,13 @@ import tempfile
 from urllib.parse import parse_qs, quote, urlparse
 
 import dlt
-import pyarrow.parquet  # type: ignore
 from dlt.common.configuration.specs import AwsCredentials
 from dlt.destinations.impl.clickhouse.configuration import (
     ClickHouseCredentials,
 )
 
+from ingestr.src.loader import load_dlt_file
+
 
 class GenericSqlDestination:
     def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:
@@ -59,9 +60,14 @@ class BigQueryDestination:
                 base64.b64decode(credentials_base64[0]).decode("utf-8")
             )
 
+        project_id = None
+        if source_fields.hostname:
+            project_id = source_fields.hostname
+
         return dlt.destinations.bigquery(
             credentials=credentials,  # type: ignore
             location=location,
+            project_id=project_id,
             **kwargs,
         )
 
@@ -184,11 +190,9 @@ class CsvDestination(GenericSqlDestination):
         if output_path.count("/") > 1:
             os.makedirs(os.path.dirname(output_path), exist_ok=True)
 
-        table = pyarrow.parquet.read_table(first_file_path)
-        rows = table.to_pylist()
         with open(output_path, "w", newline="") as csv_file:
             csv_writer = None
-            for row in rows:
+            for row in load_dlt_file(first_file_path):
                 row = filter_keys(row)
                 if csv_writer is None:
                     csv_writer = csv.DictWriter(csv_file, fieldnames=row.keys())

ingestr/src/factory.py

@@ -66,6 +66,7 @@ SQL_SOURCE_SCHEMES = [
     "oracle",
     "oracle+cx_oracle",
     "hana",
+    "clickhouse",
 ]
 
 

ingestr/src/loader.py

@@ -0,0 +1,69 @@
+import csv
+import gzip
+import json
+import subprocess
+from contextlib import contextmanager
+from typing import Generator
+
+from pyarrow.parquet import ParquetFile  # type: ignore
+
+PARQUET_BATCH_SIZE = 64
+
+
+class UnsupportedLoaderFileFormat(Exception):
+    pass
+
+
+def load_dlt_file(filepath: str) -> Generator:
+    """
+    load_dlt_file reads dlt loader files. It handles different loader file formats
+    automatically. It returns a generator that yield data items as a python dict
+    """
+    result = subprocess.run(
+        ["file", "-b", filepath],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+
+    filetype = result.stdout.strip()
+    with factory(filetype, filepath) as reader:
+        yield from reader
+
+
+def factory(filetype: str, filepath: str):
+    # ???(turtledev): can dlt produce non-gizpped jsonl files?
+    if filetype.startswith("gzip"):
+        return jsonlfile(filepath)
+    elif filetype.startswith("CSV"):
+        return csvfile(filepath)
+    elif filetype.startswith("Apache Parquet"):
+        return parquetfile(filepath)
+    else:
+        raise UnsupportedLoaderFileFormat(filetype)
+
+
+@contextmanager
+def jsonlfile(filepath: str):
+    def reader(fd):
+        for line in fd:
+            yield json.loads(line.decode().strip())
+
+    with gzip.open(filepath) as fd:
+        yield reader(fd)
+
+
+@contextmanager
+def csvfile(filepath: str):
+    with open(filepath, "r") as fd:
+        yield csv.DictReader(fd)
+
+
+@contextmanager
+def parquetfile(filepath: str):
+    def reader(pf: ParquetFile):
+        for batch in pf.iter_batches(PARQUET_BATCH_SIZE):
+            yield from batch.to_pylist()
+
+    with open(filepath, "rb") as fd:
+        yield reader(ParquetFile(fd))

ingestr/src/sources.py

@@ -133,6 +133,11 @@ class SqlSource:
         if uri.startswith("mysql://"):
             uri = uri.replace("mysql://", "mysql+pymysql://")
 
+        if uri.startswith("clickhouse://"):
+            uri = uri.replace("clickhouse://", "clickhouse+native://")
+            if "secure=" not in uri:
+                uri += "?secure=1"
+
         query_adapters = []
         if kwargs.get("sql_limit"):
             query_adapters.append(

ingestr/src/version.py

@@ -1 +1,6 @@
-__version__ = "0.13.3"
+try:
+    from ingestr.src import buildinfo  # type: ignore
+
+    __version__ = buildinfo.version.lstrip("v")
+except ImportError:
+    __version__ = "0.0.0-dev"

{ingestr-0.13.3.dist-info → ingestr-0.13.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.3
+Version: 0.13.5
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -132,7 +132,6 @@ Pull requests are welcome. However, please open an issue first to discuss what y
 > After cloning `ingestr` make sure to run `make setup` to install githooks.
 
 ## Supported sources & destinations
-
 <table>
     <tr>
         <th></th>
@@ -142,33 +141,28 @@ Pull requests are welcome. However, please open an issue first to discuss what y
     <tr>
         <td colspan="3" style='text-align:center;'><strong>Databases</strong></td>
     </tr>
-    <tr>
-        <td>Postgres</td>
-        <td>✅</td>
-        <td>✅</td>
-    </tr>
     <tr>
         <td>BigQuery</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
     <tr>
-        <td>Snowflake</td>
-        <td>✅</td>
+        <td>ClickHouse</td>
+        <td>❌</td>
         <td>✅</td>
     </tr>
     <tr>
-        <td>Redshift</td>
+        <td>Databricks</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
     <tr>
-        <td>Databricks</td>
+        <td>DuckDB</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
     <tr>
-        <td>DuckDB</td>
+        <td>Local CSV file</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
@@ -178,12 +172,12 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
     </tr>
     <tr>
-        <td>Local CSV file</td>
-        <td>✅</td>
+        <td>MongoDB</td>
         <td>✅</td>
+        <td>❌</td>
     </tr>
     <tr>
-        <td>MongoDB</td>
+        <td>MySQL</td>
         <td>✅</td>
         <td>❌</td>
     </tr>
@@ -192,18 +186,28 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>❌</td>
     </tr>
+    <tr>
+        <td>Postgres</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+     <tr>
+        <td>Redshift</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
     <tr>
         <td>SAP Hana</td>
         <td>✅</td>
         <td>❌</td>
     </tr>
     <tr>
-        <td>SQLite</td>
+        <td>Snowflake</td>
+        <td>✅</td>
         <td>✅</td>
-        <td>❌</td>
     </tr>
     <tr>
-        <td>MySQL</td>
+        <td>SQLite</td>
         <td>✅</td>
         <td>❌</td>
     </tr>
@@ -218,11 +222,21 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Apache Kafka</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>AppsFlyer</td>
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>App Store</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Asana</td>
         <td>✅</td>
@@ -243,6 +257,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Github</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Gorgias</td>
         <td>✅</td>
@@ -254,7 +273,17 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>-</td>
     </tr>
     <tr>
-        <td>HubSpot</td>
+        <td>Google Ads</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Google Analytics</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Intercom</td>
         <td>✅</td>
         <td>-</td>
     </tr>
@@ -263,6 +292,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>LinkedIn Ads</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Notion</td>
         <td>✅</td>
@@ -288,6 +322,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>TikTok Ads</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Zendesk</td>
         <td>✅</td>

{ingestr-0.13.3.dist-info → ingestr-0.13.5.dist-info}/RECORD

@@ -1,14 +1,15 @@
 ingestr/main.py,sha256=ufn8AcM2ID80ChUApJzYDjnQaurMXOkYfTm6GzAggSQ,24746
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=XDk_XqmU_He4sQ1brY3ceoZgpq_ZBZihz1gHW9MzqUk,1381
-ingestr/src/destinations.py,sha256=WxerdCx0gS4JveYAE-GzdJLbgP4t2QXXQhUoU3GvmLM,11194
+ingestr/src/destinations.py,sha256=vrGij4qMPCdXTMIimROWBJFqzOqCM4DFmgyubgSHejA,11279
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=b1Fg3lhTu6HoALi4AFrTOm3fbNPj5EB2lPigrjiY1so,4926
+ingestr/src/factory.py,sha256=CG_CZox_vJOYfmlCJ1FH9Ipb5LDeESCzox5ZOAO-wjs,4944
 ingestr/src/filters.py,sha256=0JQXeAr2APFMnW2sd-6BlAMWv93bXV17j8b5MM8sHmM,580
-ingestr/src/sources.py,sha256=YkBCyjI8DWovyA8CI68zOlHT4UKPsG8-UTCyuafI_WA,63075
+ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
+ingestr/src/sources.py,sha256=AxtWtqhn1OOFAYypLph2FhmD3CD7CDVBRvOIKb6mDIg,63263
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
-ingestr/src/version.py,sha256=aiCDTKDs80gZjvqiXYkxhq_MLi4Du0L9OByDr6ZEVV4,23
+ingestr/src/version.py,sha256=l6zVm0GMMwnBlIOONWc6snhko9d8-HO1y6Jj1T1vsiQ,158
 ingestr/src/adjust/__init__.py,sha256=ULjtJqrNS6XDvUyGl0tjl12-tLyXlCgeFe2icTbtu3Q,3255
 ingestr/src/adjust/adjust_helpers.py,sha256=av97NPSn-hQtTbAC0vUSCAWYePmOiG5R-DGdMssm7FQ,3646
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
@@ -101,8 +102,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.3.dist-info/METADATA,sha256=mVT47j7eOqecG3fnDHEkWzqmM4QHWDUixAWKpkGsoEk,8252
-ingestr-0.13.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.3.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.3.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.3.dist-info/RECORD,,
+ingestr-0.13.5.dist-info/METADATA,sha256=5kk8N8xrnWAJvOL4n2Mh1Xv7sXOn6iJyybHVBEGNRcU,8956
+ingestr-0.13.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.5.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.5.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.5.dist-info/RECORD,,