ingestr 0.10.0rc3__py3-none-any.whl → 0.10.0rc5__py3-none-any.whl

ingestr/main.py

@@ -280,6 +280,13 @@ def ingest(
             envvar="SQL_LIMIT",
         ),
     ] = None,  # type: ignore
+    sql_exclude_columns: Annotated[
+        Optional[list[str]],
+        typer.Option(
+            help="The columns to exclude from the source table",
+            envvar="SQL_EXCLUDE_COLUMNS",
+        ),
+    ] = [],  # type: ignore
 ):
     import hashlib
     import tempfile
@@ -348,6 +355,13 @@ def ingest(
         },
     )
 
+    clean_sql_exclude_columns = []
+    if sql_exclude_columns:
+        for col in sql_exclude_columns:
+            for possible_col in col.split(","):
+                clean_sql_exclude_columns.append(possible_col.strip())
+        sql_exclude_columns = clean_sql_exclude_columns
+
     dlt.config["data_writer.buffer_max_items"] = page_size
     dlt.config["data_writer.file_max_items"] = loader_file_size
     dlt.config["extract.workers"] = extract_parallelism
@@ -459,9 +473,6 @@ def ingest(
         if factory.source_scheme == "sqlite":
             source_table = "main." + source_table.split(".")[-1]
 
-        if factory.destination_scheme == "bigquery":
-            dlt.config["destination.bigquery.autodetect_schema"] = True
-
         dlt_source = source.dlt_source(
             uri=source_uri,
             table=source_table,
@@ -473,6 +484,7 @@ def ingest(
             page_size=page_size,
             sql_reflection_level=sql_reflection_level.value,
             sql_limit=sql_limit,
+            sql_exclude_columns=sql_exclude_columns,
         )
 
         dlt_source.add_map(cast_set_to_list)

ingestr/src/filters.py

@@ -1,3 +1,6 @@
+from dlt.common.libs.sql_alchemy import Table
+
+
 def cast_set_to_list(row):
     # this handles just the sqlalchemy backend for now
     if isinstance(row, dict):
@@ -5,3 +8,14 @@ def cast_set_to_list(row):
             if isinstance(row[key], set):
                 row[key] = list(row[key])
     return row
+
+
+def table_adapter_exclude_columns(cols: list[str]):
+    print("given cols", cols)
+
+    def excluder(table: Table):
+        cols_to_remove = [col for col in table._columns if col.name in cols]  # type: ignore
+        for col in cols_to_remove:
+            table._columns.remove(col)  # type: ignore
+
+    return excluder

ingestr/src/sources.py

@@ -23,6 +23,7 @@ from ingestr.src.arrow import memory_mapped_arrow
 from ingestr.src.chess import source
 from ingestr.src.facebook_ads import facebook_ads_source, facebook_insights_source
 from ingestr.src.filesystem import readers
+from ingestr.src.filters import table_adapter_exclude_columns
 from ingestr.src.google_sheets import google_spreadsheet
 from ingestr.src.gorgias import gorgias_source
 from ingestr.src.hubspot import hubspot
@@ -92,6 +93,9 @@ class SqlSource:
             reflection_level=reflection_level,
             query_adapter_callback=query_adapter_callback,
             type_adapter_callback=type_adapter_callback,
+            table_adapter_callback=table_adapter_exclude_columns(
+                kwargs.get("sql_exclude_columns", [])
+            ),
         )
 
         return table_instance

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.10.0rc3"
+__version__ = "0.10.0rc5"

{ingestr-0.10.0rc3.dist-info → ingestr-0.10.0rc5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.10.0rc3
+Version: 0.10.0rc5
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.10.0rc3.dist-info → ingestr-0.10.0rc5.dist-info}/RECORD

@@ -1,11 +1,11 @@
-ingestr/main.py,sha256=e_Loavjd3LDNAca4B_Ck-d1rI5QfvQoYQRicZSlV9G8,21000
+ingestr/main.py,sha256=UqSuqOMJPoq0fkcrba8D5gBfFIMKPEBEh0n19PJrP1c,21439
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/destinations.py,sha256=2SfPMjtTelPmzQmc3zNs8xGcKIPuGn_hoZFIBUuhjXI,6338
 ingestr/src/factory.py,sha256=ft81B-YJgvEROkHAZjMjTIS7IYvle-uZQv45b7-Wfk0,4947
-ingestr/src/filters.py,sha256=mHMj8Zp3_CsVLDTT1gxrkQTAsI_hheEaTsnoLpVm6Fk,242
-ingestr/src/sources.py,sha256=Kp4KN5Dt7eVGSJHPUsdbPoc-uF59zr6wXIMOc03Lo3s,34447
+ingestr/src/filters.py,sha256=9ckkhNF0lGVvSbw24VKQGXdtvxDCOfnSIFjswMjIu-A,611
+ingestr/src/sources.py,sha256=lvp_xAsa0EUvzEq0Txbc4gNe-JPAn_47jyHdSHKwLOQ,34644
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
-ingestr/src/version.py,sha256=ElqoMRDsqGwCMiLiVG7t6YPRo495Y_ow7PO763ym2QI,26
+ingestr/src/version.py,sha256=FipU1D3j5OsHv2DjFcCBccEi2EiXUNKnXRgNJjIA7Vo,26
 ingestr/src/adjust/__init__.py,sha256=oTM7XozDcMuUiCZ0w4gWEBXuCCtMZ0iBfkKdd2pVa1E,3007
 ingestr/src/adjust/adjust_helpers.py,sha256=-tmmxy9k3wms-ZEIgxmlp2cAQ2X_O1lgjY1128bbMu4,3224
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
@@ -70,8 +70,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.10.0rc3.dist-info/METADATA,sha256=qDA8J1GC-BkSgxF27QrQjaeZgpi_QhL2PmwSs8qtGJM,7063
-ingestr-0.10.0rc3.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
-ingestr-0.10.0rc3.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.10.0rc3.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.10.0rc3.dist-info/RECORD,,
+ingestr-0.10.0rc5.dist-info/METADATA,sha256=GdCvj3lzj-P3rQ0kW27B18lP_XeikycDnOSU-Hp01wQ,7063
+ingestr-0.10.0rc5.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
+ingestr-0.10.0rc5.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.10.0rc5.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.10.0rc5.dist-info/RECORD,,