ingestr 0.10.0rc0__tar.gz → 0.10.0rc1__tar.gz

{ingestr-0.10.0rc0 → ingestr-0.10.0rc1}/Makefile

@@ -15,7 +15,7 @@ deps-ci:
 	uv pip install --system -r requirements-dev.txt
 
 test-ci:
-	pytest -n logical -x -rP -vv --tb=short --cov=ingestr --no-cov-on-fail
+	pytest -n auto -x -rP -vv --tb=short --cov=ingestr --no-cov-on-fail
 
 test: venv
 	. venv/bin/activate; $(MAKE) test-ci

{ingestr-0.10.0rc0 → ingestr-0.10.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.10.0rc0
+Version: 0.10.0rc1
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.10.0rc0 → ingestr-0.10.0rc1}/ingestr/main.py

@@ -112,6 +112,12 @@ class SchemaNaming(str, Enum):
     direct = "direct"
 
 
+class SqlReflectionLevel(str, Enum):
+    minimal = "minimal"
+    full = "full"
+    full_with_precision = "full_with_precision"
+
+
 @app.command()
 def ingest(
     source_uri: Annotated[
@@ -259,6 +265,20 @@ def ingest(
             envvar="EXTRACT_PARALLELISM",
         ),
     ] = 5,  # type: ignore
+    sql_reflection_level: Annotated[
+        SqlReflectionLevel,
+        typer.Option(
+            help="The reflection level to use when reflecting the table schema from the source",
+            envvar="SQL_REFLECTION_LEVEL",
+        ),
+    ] = SqlReflectionLevel.full,  # type: ignore
+    sql_limit: Annotated[
+        Optional[int],
+        typer.Option(
+            help="The limit to use when fetching data from the source",
+            envvar="SQL_LIMIT",
+        ),
+    ] = None,  # type: ignore
 ):
     import hashlib
     import tempfile
@@ -447,6 +467,8 @@ def ingest(
             interval_end=interval_end,
             sql_backend=sql_backend.value,
             page_size=page_size,
+            sql_reflection_level=sql_reflection_level.value,
+            sql_limit=sql_limit,
         )
 
         if original_incremental_strategy == IncrementalStrategy.delete_insert:
@@ -494,7 +516,6 @@ def ingest(
         elapsed = end_time - start_time
         elapsedHuman = f"in {humanize.precisedelta(elapsed)}"
 
-        # remove the pipelines_dir folder if it was created by ingestr
         if is_pipelines_dir_temp:
             import shutil
 

{ingestr-0.10.0rc0 → ingestr-0.10.0rc1}/ingestr/src/sources.py

@@ -67,6 +67,14 @@ class SqlSource:
         if uri.startswith("mysql://"):
             uri = uri.replace("mysql://", "mysql+pymysql://")
 
+        reflection_level = kwargs.get("sql_reflection_level")
+
+        query_adapter_callback = None
+        if kwargs.get("sql_limit"):
+
+            def query_adapter_callback(query, table):
+                return query.limit(kwargs.get("sql_limit"))
+
         table_instance = self.table_builder(
             credentials=ConnectionStringCredentials(uri),
             schema=table_fields.dataset,
@@ -74,6 +82,8 @@ class SqlSource:
             incremental=incremental,
             backend=kwargs.get("sql_backend", "sqlalchemy"),
             chunk_size=kwargs.get("page_size", None),
+            reflection_level=reflection_level,
+            query_adapter_callback=query_adapter_callback,
         )
 
         return table_instance

ingestr-0.10.0rc1/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.10.0rc1"

ingestr-0.10.0rc0/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.10.0rc0"