ingestr 0.10.0rc4__tar.gz → 0.10.1__tar.gz

{ingestr-0.10.0rc4 → ingestr-0.10.1}/.gitignore

@@ -19,3 +19,4 @@ node_modules
 *.duckdb.wal
 *.db
 *.db.wal
+*.sql

{ingestr-0.10.0rc4 → ingestr-0.10.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.10.0rc4
+Version: 0.10.1
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.10.0rc4 → ingestr-0.10.1}/ingestr/main.py

@@ -280,6 +280,13 @@ def ingest(
             envvar="SQL_LIMIT",
         ),
     ] = None,  # type: ignore
+    sql_exclude_columns: Annotated[
+        Optional[list[str]],
+        typer.Option(
+            help="The columns to exclude from the source table",
+            envvar="SQL_EXCLUDE_COLUMNS",
+        ),
+    ] = [],  # type: ignore
 ):
     import hashlib
     import tempfile
@@ -341,6 +348,14 @@ def ingest(
             )
             raise typer.Abort()
 
+    def run_on_resource(source, executable):
+        if hasattr(source, "selected_resources") and source.selected_resources:
+            resource_names = list(source.selected_resources.keys())
+            for res in resource_names:
+                executable(source.resources[res])
+        else:
+            executable(source)
+
     track(
         "command_triggered",
         {
@@ -348,6 +363,13 @@ def ingest(
         },
     )
 
+    clean_sql_exclude_columns = []
+    if sql_exclude_columns:
+        for col in sql_exclude_columns:
+            for possible_col in col.split(","):
+                clean_sql_exclude_columns.append(possible_col.strip())
+        sql_exclude_columns = clean_sql_exclude_columns
+
     dlt.config["data_writer.buffer_max_items"] = page_size
     dlt.config["data_writer.file_max_items"] = loader_file_size
     dlt.config["extract.workers"] = extract_parallelism
@@ -459,9 +481,6 @@ def ingest(
         if factory.source_scheme == "sqlite":
             source_table = "main." + source_table.split(".")[-1]
 
-        if factory.destination_scheme == "bigquery" and loader_file_format is LoaderFileFormat.parquet:
-            dlt.config["destination.bigquery.autodetect_schema"] = True
-
         dlt_source = source.dlt_source(
             uri=source_uri,
             table=source_table,
@@ -473,12 +492,23 @@ def ingest(
             page_size=page_size,
             sql_reflection_level=sql_reflection_level.value,
             sql_limit=sql_limit,
+            sql_exclude_columns=sql_exclude_columns,
         )
 
-        dlt_source.add_map(cast_set_to_list)
+        run_on_resource(dlt_source, lambda x: x.add_map(cast_set_to_list))
+
+        def col_h(x):
+            if column_hints:
+                x.apply_hints(columns=column_hints)
+
+        run_on_resource(dlt_source, col_h)
 
         if original_incremental_strategy == IncrementalStrategy.delete_insert:
-            dlt_source.incremental.primary_key = ()
+
+            def set_primary_key(x):
+                x.incremental.primary_key = ()
+
+            run_on_resource(dlt_source, set_primary_key)
 
         if (
             factory.destination_scheme in PARQUET_SUPPORTED_DESTINATIONS
@@ -510,7 +540,6 @@ def ingest(
             loader_file_format=(
                 loader_file_format.value if loader_file_format is not None else None  # type: ignore
             ),  # type: ignore
-            columns=column_hints,
         )
 
         report_errors(run_info)

{ingestr-0.10.0rc4 → ingestr-0.10.1}/ingestr/src/adjust/__init__.py

@@ -82,7 +82,7 @@ def adjust_source(
             type_hints[metric] = KNOWN_TYPE_HINTS[metric]
 
     @dlt.resource(
-        write_disposition={"disposition": "merge", "strategy": "delete+insert"},
+        write_disposition={"disposition": "merge", "strategy": "delete-insert"},
         merge_key=merge_key,
         primary_key=dimensions,
         columns=type_hints,

ingestr-0.10.1/ingestr/src/filters.py

@@ -0,0 +1,19 @@
+from dlt.common.libs.sql_alchemy import Table
+
+
+def cast_set_to_list(row):
+    # this handles just the sqlalchemy backend for now
+    if isinstance(row, dict):
+        for key in row.keys():
+            if isinstance(row[key], set):
+                row[key] = list(row[key])
+    return row
+
+
+def table_adapter_exclude_columns(cols: list[str]):
+    def excluder(table: Table):
+        cols_to_remove = [col for col in table._columns if col.name in cols]  # type: ignore
+        for col in cols_to_remove:
+            table._columns.remove(col)  # type: ignore
+
+    return excluder

{ingestr-0.10.0rc4 → ingestr-0.10.1}/ingestr/src/sources.py

@@ -23,6 +23,7 @@ from ingestr.src.arrow import memory_mapped_arrow
 from ingestr.src.chess import source
 from ingestr.src.facebook_ads import facebook_ads_source, facebook_insights_source
 from ingestr.src.filesystem import readers
+from ingestr.src.filters import table_adapter_exclude_columns
 from ingestr.src.google_sheets import google_spreadsheet
 from ingestr.src.gorgias import gorgias_source
 from ingestr.src.hubspot import hubspot
@@ -75,14 +76,17 @@ class SqlSource:
         if kwargs.get("sql_limit"):
 
             def query_adapter_callback(query, table):
-                return query.limit(kwargs.get("sql_limit"))
+                query = query.limit(kwargs.get("sql_limit"))
+                if kwargs.get("incremental_key"):
+                    query = query.order_by(kwargs.get("incremental_key"))
+                return query
 
         def type_adapter_callback(sql_type):
             if isinstance(sql_type, mysql.SET):
                 return sa.JSON
             return sql_type
 
-        table_instance = self.table_builder(
+        builder_res = self.table_builder(
             credentials=ConnectionStringCredentials(uri),
             schema=table_fields.dataset,
             table=table_fields.table,
@@ -92,9 +96,12 @@ class SqlSource:
             reflection_level=reflection_level,
             query_adapter_callback=query_adapter_callback,
             type_adapter_callback=type_adapter_callback,
+            table_adapter_callback=table_adapter_exclude_columns(
+                kwargs.get("sql_exclude_columns", [])
+            ),
         )
 
-        return table_instance
+        return builder_res
 
 
 class ArrowMemoryMappedSource:
@@ -740,7 +747,7 @@ class KafkaSource:
 
 class AdjustSource:
     def handles_incrementality(self) -> bool:
-        return False
+        return True
 
     def dlt_source(self, uri: str, table: str, **kwargs):
         if kwargs.get("incremental_key") and not table.startswith("custom:"):
@@ -802,7 +809,7 @@ class AdjustSource:
                 filters_raw = fields[3]
                 filters = parse_filters(filters_raw)
 
-        return adjust_source(
+        src = adjust_source(
             start_date=start_date,
             end_date=end_date,
             api_key=api_key[0],
@@ -810,7 +817,9 @@ class AdjustSource:
             metrics=metrics,
             merge_key=kwargs.get("merge_key"),
             filters=filters,
-        ).with_resources(table)
+        )
+
+        return src.with_resources(table)
 
 
 class AppsflyerSource:

ingestr-0.10.1/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.10.1"

ingestr-0.10.0rc4/ingestr/src/filters.py

@@ -1,7 +0,0 @@
-def cast_set_to_list(row):
-    # this handles just the sqlalchemy backend for now
-    if isinstance(row, dict):
-        for key in row.keys():
-            if isinstance(row[key], set):
-                row[key] = list(row[key])
-    return row

ingestr-0.10.0rc4/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.10.0rc4"