ingestr 0.10.0rc1__py3-none-any.whl → 0.10.0rc3__py3-none-any.whl

ingestr/main.py

@@ -8,6 +8,7 @@ from rich.console import Console
 from rich.status import Status
 from typing_extensions import Annotated
 
+from ingestr.src.filters import cast_set_to_list
 from ingestr.src.telemetry.event import track
 
 app = typer.Typer(
@@ -458,6 +459,9 @@ def ingest(
         if factory.source_scheme == "sqlite":
             source_table = "main." + source_table.split(".")[-1]
 
+        if factory.destination_scheme == "bigquery":
+            dlt.config["destination.bigquery.autodetect_schema"] = True
+
         dlt_source = source.dlt_source(
             uri=source_uri,
             table=source_table,
@@ -471,6 +475,8 @@ def ingest(
             sql_limit=sql_limit,
         )
 
+        dlt_source.add_map(cast_set_to_list)
+
         if original_incremental_strategy == IncrementalStrategy.delete_insert:
             dlt_source.incremental.primary_key = ()
 
@@ -501,9 +507,9 @@ def ingest(
             ),
             write_disposition=write_disposition,  # type: ignore
             primary_key=(primary_key if primary_key and len(primary_key) > 0 else None),  # type: ignore
-            loader_file_format=loader_file_format.value
-            if loader_file_format is not None
-            else None,  # type: ignore
+            loader_file_format=(
+                loader_file_format.value if loader_file_format is not None else None  # type: ignore
+            ),  # type: ignore
             columns=column_hints,
         )
 

ingestr/src/filters.py

@@ -0,0 +1,7 @@
+def cast_set_to_list(row):
+    # this handles just the sqlalchemy backend for now
+    if isinstance(row, dict):
+        for key in row.keys():
+            if isinstance(row[key], set):
+                row[key] = list(row[key])
+    return row

ingestr/src/sources.py

@@ -12,6 +12,8 @@ from dlt.common.time import ensure_pendulum_datetime
 from dlt.common.typing import TSecretStrValue
 from dlt.sources.credentials import ConnectionStringCredentials
 from dlt.sources.sql_database import sql_table
+from sqlalchemy import types as sa
+from sqlalchemy.dialects import mysql
 
 from ingestr.src.adjust import REQUIRED_CUSTOM_DIMENSIONS, adjust_source
 from ingestr.src.adjust.adjust_helpers import parse_filters
@@ -75,6 +77,11 @@ class SqlSource:
             def query_adapter_callback(query, table):
                 return query.limit(kwargs.get("sql_limit"))
 
+        def type_adapter_callback(sql_type):
+            if isinstance(sql_type, mysql.SET):
+                return sa.JSON
+            return sql_type
+
         table_instance = self.table_builder(
             credentials=ConnectionStringCredentials(uri),
             schema=table_fields.dataset,
@@ -84,6 +91,7 @@ class SqlSource:
             chunk_size=kwargs.get("page_size", None),
             reflection_level=reflection_level,
             query_adapter_callback=query_adapter_callback,
+            type_adapter_callback=type_adapter_callback,
         )
 
         return table_instance

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.10.0rc1"
+__version__ = "0.10.0rc3"

{ingestr-0.10.0rc1.dist-info → ingestr-0.10.0rc3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.10.0rc1
+Version: 0.10.0rc3
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.10.0rc1.dist-info → ingestr-0.10.0rc3.dist-info}/RECORD

@@ -1,10 +1,11 @@
-ingestr/main.py,sha256=F38DRQ-ZjwCQaEbSWfg1cSThYMBgOnQWOd-G9rinC1s,20755
+ingestr/main.py,sha256=e_Loavjd3LDNAca4B_Ck-d1rI5QfvQoYQRicZSlV9G8,21000
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/destinations.py,sha256=2SfPMjtTelPmzQmc3zNs8xGcKIPuGn_hoZFIBUuhjXI,6338
 ingestr/src/factory.py,sha256=ft81B-YJgvEROkHAZjMjTIS7IYvle-uZQv45b7-Wfk0,4947
-ingestr/src/sources.py,sha256=ajknMdZrRdtMASrvdUEJNO7T49mKYR6owhPm0d-h-Bo,34164
+ingestr/src/filters.py,sha256=mHMj8Zp3_CsVLDTT1gxrkQTAsI_hheEaTsnoLpVm6Fk,242
+ingestr/src/sources.py,sha256=Kp4KN5Dt7eVGSJHPUsdbPoc-uF59zr6wXIMOc03Lo3s,34447
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
-ingestr/src/version.py,sha256=Tme0LgDiYpeJhYWEQJW_1BdVaxEwPjv-S0uEHbcUTRY,26
+ingestr/src/version.py,sha256=ElqoMRDsqGwCMiLiVG7t6YPRo495Y_ow7PO763ym2QI,26
 ingestr/src/adjust/__init__.py,sha256=oTM7XozDcMuUiCZ0w4gWEBXuCCtMZ0iBfkKdd2pVa1E,3007
 ingestr/src/adjust/adjust_helpers.py,sha256=-tmmxy9k3wms-ZEIgxmlp2cAQ2X_O1lgjY1128bbMu4,3224
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
@@ -69,8 +70,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.10.0rc1.dist-info/METADATA,sha256=FEWuTxIM11p3nERDqWGnj9NeaaNx4fy9bEJ4r4msuSw,7063
-ingestr-0.10.0rc1.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
-ingestr-0.10.0rc1.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.10.0rc1.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.10.0rc1.dist-info/RECORD,,
+ingestr-0.10.0rc3.dist-info/METADATA,sha256=qDA8J1GC-BkSgxF27QrQjaeZgpi_QhL2PmwSs8qtGJM,7063
+ingestr-0.10.0rc3.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
+ingestr-0.10.0rc3.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.10.0rc3.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.10.0rc3.dist-info/RECORD,,