ingestr 0.10.0rc4__tar.gz → 0.10.0rc5__tar.gz

{ingestr-0.10.0rc4 → ingestr-0.10.0rc5}/.gitignore

@@ -19,3 +19,4 @@ node_modules
 *.duckdb.wal
 *.db
 *.db.wal
+*.sql

{ingestr-0.10.0rc4 → ingestr-0.10.0rc5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.10.0rc4
+Version: 0.10.0rc5
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.10.0rc4 → ingestr-0.10.0rc5}/ingestr/main.py

@@ -280,6 +280,13 @@ def ingest(
             envvar="SQL_LIMIT",
         ),
     ] = None,  # type: ignore
+    sql_exclude_columns: Annotated[
+        Optional[list[str]],
+        typer.Option(
+            help="The columns to exclude from the source table",
+            envvar="SQL_EXCLUDE_COLUMNS",
+        ),
+    ] = [],  # type: ignore
 ):
     import hashlib
     import tempfile
@@ -348,6 +355,13 @@ def ingest(
         },
     )
 
+    clean_sql_exclude_columns = []
+    if sql_exclude_columns:
+        for col in sql_exclude_columns:
+            for possible_col in col.split(","):
+                clean_sql_exclude_columns.append(possible_col.strip())
+        sql_exclude_columns = clean_sql_exclude_columns
+
     dlt.config["data_writer.buffer_max_items"] = page_size
     dlt.config["data_writer.file_max_items"] = loader_file_size
     dlt.config["extract.workers"] = extract_parallelism
@@ -459,9 +473,6 @@ def ingest(
         if factory.source_scheme == "sqlite":
             source_table = "main." + source_table.split(".")[-1]
 
-        if factory.destination_scheme == "bigquery" and loader_file_format is LoaderFileFormat.parquet:
-            dlt.config["destination.bigquery.autodetect_schema"] = True
-
         dlt_source = source.dlt_source(
             uri=source_uri,
             table=source_table,
@@ -473,6 +484,7 @@ def ingest(
             page_size=page_size,
             sql_reflection_level=sql_reflection_level.value,
             sql_limit=sql_limit,
+            sql_exclude_columns=sql_exclude_columns,
         )
 
         dlt_source.add_map(cast_set_to_list)

ingestr-0.10.0rc5/ingestr/src/filters.py

@@ -0,0 +1,21 @@
+from dlt.common.libs.sql_alchemy import Table
+
+
+def cast_set_to_list(row):
+    # this handles just the sqlalchemy backend for now
+    if isinstance(row, dict):
+        for key in row.keys():
+            if isinstance(row[key], set):
+                row[key] = list(row[key])
+    return row
+
+
+def table_adapter_exclude_columns(cols: list[str]):
+    print("given cols", cols)
+
+    def excluder(table: Table):
+        cols_to_remove = [col for col in table._columns if col.name in cols]  # type: ignore
+        for col in cols_to_remove:
+            table._columns.remove(col)  # type: ignore
+
+    return excluder

{ingestr-0.10.0rc4 → ingestr-0.10.0rc5}/ingestr/src/sources.py

@@ -23,6 +23,7 @@ from ingestr.src.arrow import memory_mapped_arrow
 from ingestr.src.chess import source
 from ingestr.src.facebook_ads import facebook_ads_source, facebook_insights_source
 from ingestr.src.filesystem import readers
+from ingestr.src.filters import table_adapter_exclude_columns
 from ingestr.src.google_sheets import google_spreadsheet
 from ingestr.src.gorgias import gorgias_source
 from ingestr.src.hubspot import hubspot
@@ -92,6 +93,9 @@ class SqlSource:
             reflection_level=reflection_level,
             query_adapter_callback=query_adapter_callback,
             type_adapter_callback=type_adapter_callback,
+            table_adapter_callback=table_adapter_exclude_columns(
+                kwargs.get("sql_exclude_columns", [])
+            ),
         )
 
         return table_instance

ingestr-0.10.0rc5/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.10.0rc5"

ingestr-0.10.0rc4/ingestr/src/filters.py

@@ -1,7 +0,0 @@
-def cast_set_to_list(row):
-    # this handles just the sqlalchemy backend for now
-    if isinstance(row, dict):
-        for key in row.keys():
-            if isinstance(row[key], set):
-                row[key] = list(row[key])
-    return row

ingestr-0.10.0rc4/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.10.0rc4"