ingestr 0.10.0rc0__tar.gz → 0.10.0rc2__tar.gz

{ingestr-0.10.0rc0 → ingestr-0.10.0rc2}/Makefile

@@ -15,7 +15,7 @@ deps-ci:
 	uv pip install --system -r requirements-dev.txt
 
 test-ci:
-	pytest -n logical -x -rP -vv --tb=short --cov=ingestr --no-cov-on-fail
+	TESTCONTAINERS_RYUK_DISABLED=true pytest -n auto -x -rP -vv --tb=short --durations=10 --cov=ingestr --no-cov-on-fail
 
 test: venv
 	. venv/bin/activate; $(MAKE) test-ci

{ingestr-0.10.0rc0 → ingestr-0.10.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.10.0rc0
+Version: 0.10.0rc2
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.10.0rc0 → ingestr-0.10.0rc2}/ingestr/main.py

@@ -8,6 +8,7 @@ from rich.console import Console
 from rich.status import Status
 from typing_extensions import Annotated
 
+from ingestr.src.filters import cast_set_to_list
 from ingestr.src.telemetry.event import track
 
 app = typer.Typer(
@@ -112,6 +113,12 @@ class SchemaNaming(str, Enum):
     direct = "direct"
 
 
+class SqlReflectionLevel(str, Enum):
+    minimal = "minimal"
+    full = "full"
+    full_with_precision = "full_with_precision"
+
+
 @app.command()
 def ingest(
     source_uri: Annotated[
@@ -259,6 +266,20 @@ def ingest(
             envvar="EXTRACT_PARALLELISM",
         ),
     ] = 5,  # type: ignore
+    sql_reflection_level: Annotated[
+        SqlReflectionLevel,
+        typer.Option(
+            help="The reflection level to use when reflecting the table schema from the source",
+            envvar="SQL_REFLECTION_LEVEL",
+        ),
+    ] = SqlReflectionLevel.full,  # type: ignore
+    sql_limit: Annotated[
+        Optional[int],
+        typer.Option(
+            help="The limit to use when fetching data from the source",
+            envvar="SQL_LIMIT",
+        ),
+    ] = None,  # type: ignore
 ):
     import hashlib
     import tempfile
@@ -447,8 +468,12 @@ def ingest(
             interval_end=interval_end,
             sql_backend=sql_backend.value,
             page_size=page_size,
+            sql_reflection_level=sql_reflection_level.value,
+            sql_limit=sql_limit,
         )
 
+        dlt_source.add_map(cast_set_to_list)
+
         if original_incremental_strategy == IncrementalStrategy.delete_insert:
             dlt_source.incremental.primary_key = ()
 
@@ -479,9 +504,9 @@ def ingest(
             ),
             write_disposition=write_disposition,  # type: ignore
             primary_key=(primary_key if primary_key and len(primary_key) > 0 else None),  # type: ignore
-            loader_file_format=loader_file_format.value
-            if loader_file_format is not None
-            else None,  # type: ignore
+            loader_file_format=(
+                loader_file_format.value if loader_file_format is not None else None  # type: ignore
+            ),  # type: ignore
             columns=column_hints,
         )
 
@@ -494,7 +519,6 @@ def ingest(
         elapsed = end_time - start_time
         elapsedHuman = f"in {humanize.precisedelta(elapsed)}"
 
-        # remove the pipelines_dir folder if it was created by ingestr
         if is_pipelines_dir_temp:
             import shutil
 

ingestr-0.10.0rc2/ingestr/src/filters.py

@@ -0,0 +1,7 @@
+def cast_set_to_list(row):
+    # this handles just the sqlalchemy backend for now
+    if isinstance(row, dict):
+        for key in row.keys():
+            if isinstance(row[key], set):
+                row[key] = list(row[key])
+    return row

{ingestr-0.10.0rc0 → ingestr-0.10.0rc2}/ingestr/src/sources.py

@@ -12,6 +12,8 @@ from dlt.common.time import ensure_pendulum_datetime
 from dlt.common.typing import TSecretStrValue
 from dlt.sources.credentials import ConnectionStringCredentials
 from dlt.sources.sql_database import sql_table
+from sqlalchemy import types as sa
+from sqlalchemy.dialects import mysql
 
 from ingestr.src.adjust import REQUIRED_CUSTOM_DIMENSIONS, adjust_source
 from ingestr.src.adjust.adjust_helpers import parse_filters
@@ -67,6 +69,19 @@ class SqlSource:
         if uri.startswith("mysql://"):
             uri = uri.replace("mysql://", "mysql+pymysql://")
 
+        reflection_level = kwargs.get("sql_reflection_level")
+
+        query_adapter_callback = None
+        if kwargs.get("sql_limit"):
+
+            def query_adapter_callback(query, table):
+                return query.limit(kwargs.get("sql_limit"))
+
+        def type_adapter_callback(sql_type):
+            if isinstance(sql_type, mysql.SET):
+                return sa.JSON
+            return sql_type
+
         table_instance = self.table_builder(
             credentials=ConnectionStringCredentials(uri),
             schema=table_fields.dataset,
@@ -74,6 +89,9 @@ class SqlSource:
             incremental=incremental,
             backend=kwargs.get("sql_backend", "sqlalchemy"),
             chunk_size=kwargs.get("page_size", None),
+            reflection_level=reflection_level,
+            query_adapter_callback=query_adapter_callback,
+            type_adapter_callback=type_adapter_callback,
         )
 
         return table_instance

ingestr-0.10.0rc2/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.10.0rc2"

ingestr-0.10.0rc0/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.10.0rc0"