ingestr 0.10.0rc1__tar.gz → 0.10.0rc2__tar.gz

{ingestr-0.10.0rc1 → ingestr-0.10.0rc2}/Makefile

@@ -15,7 +15,7 @@ deps-ci:
 	uv pip install --system -r requirements-dev.txt
 
 test-ci:
-	pytest -n auto -x -rP -vv --tb=short --cov=ingestr --no-cov-on-fail
+	TESTCONTAINERS_RYUK_DISABLED=true pytest -n auto -x -rP -vv --tb=short --durations=10 --cov=ingestr --no-cov-on-fail
 
 test: venv
 	. venv/bin/activate; $(MAKE) test-ci

{ingestr-0.10.0rc1 → ingestr-0.10.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.10.0rc1
+Version: 0.10.0rc2
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.10.0rc1 → ingestr-0.10.0rc2}/ingestr/main.py

@@ -8,6 +8,7 @@ from rich.console import Console
 from rich.status import Status
 from typing_extensions import Annotated
 
+from ingestr.src.filters import cast_set_to_list
 from ingestr.src.telemetry.event import track
 
 app = typer.Typer(
@@ -471,6 +472,8 @@ def ingest(
             sql_limit=sql_limit,
         )
 
+        dlt_source.add_map(cast_set_to_list)
+
         if original_incremental_strategy == IncrementalStrategy.delete_insert:
             dlt_source.incremental.primary_key = ()
 
@@ -501,9 +504,9 @@ def ingest(
             ),
             write_disposition=write_disposition,  # type: ignore
             primary_key=(primary_key if primary_key and len(primary_key) > 0 else None),  # type: ignore
-            loader_file_format=loader_file_format.value
-            if loader_file_format is not None
-            else None,  # type: ignore
+            loader_file_format=(
+                loader_file_format.value if loader_file_format is not None else None  # type: ignore
+            ),  # type: ignore
             columns=column_hints,
         )
 

ingestr-0.10.0rc2/ingestr/src/filters.py

@@ -0,0 +1,7 @@
+def cast_set_to_list(row):
+    # this handles just the sqlalchemy backend for now
+    if isinstance(row, dict):
+        for key in row.keys():
+            if isinstance(row[key], set):
+                row[key] = list(row[key])
+    return row

{ingestr-0.10.0rc1 → ingestr-0.10.0rc2}/ingestr/src/sources.py

@@ -12,6 +12,8 @@ from dlt.common.time import ensure_pendulum_datetime
 from dlt.common.typing import TSecretStrValue
 from dlt.sources.credentials import ConnectionStringCredentials
 from dlt.sources.sql_database import sql_table
+from sqlalchemy import types as sa
+from sqlalchemy.dialects import mysql
 
 from ingestr.src.adjust import REQUIRED_CUSTOM_DIMENSIONS, adjust_source
 from ingestr.src.adjust.adjust_helpers import parse_filters
@@ -75,6 +77,11 @@ class SqlSource:
             def query_adapter_callback(query, table):
                 return query.limit(kwargs.get("sql_limit"))
 
+        def type_adapter_callback(sql_type):
+            if isinstance(sql_type, mysql.SET):
+                return sa.JSON
+            return sql_type
+
         table_instance = self.table_builder(
             credentials=ConnectionStringCredentials(uri),
             schema=table_fields.dataset,
@@ -84,6 +91,7 @@ class SqlSource:
             chunk_size=kwargs.get("page_size", None),
             reflection_level=reflection_level,
             query_adapter_callback=query_adapter_callback,
+            type_adapter_callback=type_adapter_callback,
         )
 
         return table_instance

ingestr-0.10.0rc2/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.10.0rc2"

ingestr-0.10.0rc1/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.10.0rc1"