ingestr 0.10.0rc5__tar.gz → 0.10.1__tar.gz

{ingestr-0.10.0rc5 → ingestr-0.10.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.10.0rc5
+Version: 0.10.1
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.10.0rc5 → ingestr-0.10.1}/ingestr/main.py

@@ -348,6 +348,14 @@ def ingest(
             )
             raise typer.Abort()
 
+    def run_on_resource(source, executable):
+        if hasattr(source, "selected_resources") and source.selected_resources:
+            resource_names = list(source.selected_resources.keys())
+            for res in resource_names:
+                executable(source.resources[res])
+        else:
+            executable(source)
+
     track(
         "command_triggered",
         {
@@ -487,10 +495,20 @@ def ingest(
             sql_exclude_columns=sql_exclude_columns,
         )
 
-        dlt_source.add_map(cast_set_to_list)
+        run_on_resource(dlt_source, lambda x: x.add_map(cast_set_to_list))
+
+        def col_h(x):
+            if column_hints:
+                x.apply_hints(columns=column_hints)
+
+        run_on_resource(dlt_source, col_h)
 
         if original_incremental_strategy == IncrementalStrategy.delete_insert:
-            dlt_source.incremental.primary_key = ()
+
+            def set_primary_key(x):
+                x.incremental.primary_key = ()
+
+            run_on_resource(dlt_source, set_primary_key)
 
         if (
             factory.destination_scheme in PARQUET_SUPPORTED_DESTINATIONS
@@ -522,7 +540,6 @@ def ingest(
             loader_file_format=(
                 loader_file_format.value if loader_file_format is not None else None  # type: ignore
             ),  # type: ignore
-            columns=column_hints,
         )
 
         report_errors(run_info)

{ingestr-0.10.0rc5 → ingestr-0.10.1}/ingestr/src/adjust/__init__.py

@@ -82,7 +82,7 @@ def adjust_source(
             type_hints[metric] = KNOWN_TYPE_HINTS[metric]
 
     @dlt.resource(
-        write_disposition={"disposition": "merge", "strategy": "delete+insert"},
+        write_disposition={"disposition": "merge", "strategy": "delete-insert"},
         merge_key=merge_key,
         primary_key=dimensions,
         columns=type_hints,

{ingestr-0.10.0rc5 → ingestr-0.10.1}/ingestr/src/filters.py

@@ -11,8 +11,6 @@ def cast_set_to_list(row):
 
 
 def table_adapter_exclude_columns(cols: list[str]):
-    print("given cols", cols)
-
     def excluder(table: Table):
         cols_to_remove = [col for col in table._columns if col.name in cols]  # type: ignore
         for col in cols_to_remove:

{ingestr-0.10.0rc5 → ingestr-0.10.1}/ingestr/src/sources.py

@@ -76,14 +76,17 @@ class SqlSource:
         if kwargs.get("sql_limit"):
 
             def query_adapter_callback(query, table):
-                return query.limit(kwargs.get("sql_limit"))
+                query = query.limit(kwargs.get("sql_limit"))
+                if kwargs.get("incremental_key"):
+                    query = query.order_by(kwargs.get("incremental_key"))
+                return query
 
         def type_adapter_callback(sql_type):
             if isinstance(sql_type, mysql.SET):
                 return sa.JSON
             return sql_type
 
-        table_instance = self.table_builder(
+        builder_res = self.table_builder(
             credentials=ConnectionStringCredentials(uri),
             schema=table_fields.dataset,
             table=table_fields.table,
@@ -98,7 +101,7 @@ class SqlSource:
             ),
         )
 
-        return table_instance
+        return builder_res
 
 
 class ArrowMemoryMappedSource:
@@ -744,7 +747,7 @@ class KafkaSource:
 
 class AdjustSource:
     def handles_incrementality(self) -> bool:
-        return False
+        return True
 
     def dlt_source(self, uri: str, table: str, **kwargs):
         if kwargs.get("incremental_key") and not table.startswith("custom:"):
@@ -806,7 +809,7 @@ class AdjustSource:
                 filters_raw = fields[3]
                 filters = parse_filters(filters_raw)
 
-        return adjust_source(
+        src = adjust_source(
             start_date=start_date,
             end_date=end_date,
             api_key=api_key[0],
@@ -814,7 +817,9 @@ class AdjustSource:
             metrics=metrics,
             merge_key=kwargs.get("merge_key"),
             filters=filters,
-        ).with_resources(table)
+        )
+
+        return src.with_resources(table)
 
 
 class AppsflyerSource:

ingestr-0.10.1/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.10.1"

ingestr-0.10.0rc5/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.10.0rc5"