ingestr 0.10.0rc0__py3-none-any.whl → 0.10.0rc1__py3-none-any.whl

ingestr/main.py

@@ -112,6 +112,12 @@ class SchemaNaming(str, Enum):
     direct = "direct"
 
 
+class SqlReflectionLevel(str, Enum):
+    minimal = "minimal"
+    full = "full"
+    full_with_precision = "full_with_precision"
+
+
 @app.command()
 def ingest(
     source_uri: Annotated[
@@ -259,6 +265,20 @@ def ingest(
             envvar="EXTRACT_PARALLELISM",
         ),
     ] = 5,  # type: ignore
+    sql_reflection_level: Annotated[
+        SqlReflectionLevel,
+        typer.Option(
+            help="The reflection level to use when reflecting the table schema from the source",
+            envvar="SQL_REFLECTION_LEVEL",
+        ),
+    ] = SqlReflectionLevel.full,  # type: ignore
+    sql_limit: Annotated[
+        Optional[int],
+        typer.Option(
+            help="The limit to use when fetching data from the source",
+            envvar="SQL_LIMIT",
+        ),
+    ] = None,  # type: ignore
 ):
     import hashlib
     import tempfile
@@ -447,6 +467,8 @@ def ingest(
             interval_end=interval_end,
             sql_backend=sql_backend.value,
             page_size=page_size,
+            sql_reflection_level=sql_reflection_level.value,
+            sql_limit=sql_limit,
         )
 
         if original_incremental_strategy == IncrementalStrategy.delete_insert:
@@ -494,7 +516,6 @@ def ingest(
         elapsed = end_time - start_time
         elapsedHuman = f"in {humanize.precisedelta(elapsed)}"
 
-        # remove the pipelines_dir folder if it was created by ingestr
         if is_pipelines_dir_temp:
             import shutil
 

ingestr/src/sources.py

@@ -67,6 +67,14 @@ class SqlSource:
         if uri.startswith("mysql://"):
             uri = uri.replace("mysql://", "mysql+pymysql://")
 
+        reflection_level = kwargs.get("sql_reflection_level")
+
+        query_adapter_callback = None
+        if kwargs.get("sql_limit"):
+
+            def query_adapter_callback(query, table):
+                return query.limit(kwargs.get("sql_limit"))
+
         table_instance = self.table_builder(
             credentials=ConnectionStringCredentials(uri),
             schema=table_fields.dataset,
@@ -74,6 +82,8 @@ class SqlSource:
             incremental=incremental,
             backend=kwargs.get("sql_backend", "sqlalchemy"),
             chunk_size=kwargs.get("page_size", None),
+            reflection_level=reflection_level,
+            query_adapter_callback=query_adapter_callback,
         )
 
         return table_instance

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.10.0rc0"
+__version__ = "0.10.0rc1"

{ingestr-0.10.0rc0.dist-info → ingestr-0.10.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.10.0rc0
+Version: 0.10.0rc1
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.10.0rc0.dist-info → ingestr-0.10.0rc1.dist-info}/RECORD

@@ -1,10 +1,10 @@
-ingestr/main.py,sha256=O0FnyiWBjWu-Inl56EVtdHVWOggTdeEgjKbfp7zT1Kc,20100
+ingestr/main.py,sha256=F38DRQ-ZjwCQaEbSWfg1cSThYMBgOnQWOd-G9rinC1s,20755
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/destinations.py,sha256=2SfPMjtTelPmzQmc3zNs8xGcKIPuGn_hoZFIBUuhjXI,6338
 ingestr/src/factory.py,sha256=ft81B-YJgvEROkHAZjMjTIS7IYvle-uZQv45b7-Wfk0,4947
-ingestr/src/sources.py,sha256=t_t4xK-bC1dBf7w1axcopuzIERzPHIHZ_QLiEb7BOhQ,33805
+ingestr/src/sources.py,sha256=ajknMdZrRdtMASrvdUEJNO7T49mKYR6owhPm0d-h-Bo,34164
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
-ingestr/src/version.py,sha256=xTnym0lQwATiKdBCgZ51qs6XA-BvYKGOI8vHCckrPzk,26
+ingestr/src/version.py,sha256=Tme0LgDiYpeJhYWEQJW_1BdVaxEwPjv-S0uEHbcUTRY,26
 ingestr/src/adjust/__init__.py,sha256=oTM7XozDcMuUiCZ0w4gWEBXuCCtMZ0iBfkKdd2pVa1E,3007
 ingestr/src/adjust/adjust_helpers.py,sha256=-tmmxy9k3wms-ZEIgxmlp2cAQ2X_O1lgjY1128bbMu4,3224
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
@@ -69,8 +69,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.10.0rc0.dist-info/METADATA,sha256=cRvsEF34onQqsvTFbiaU9w3mb6digQL1iLG4BOi3HfQ,7063
-ingestr-0.10.0rc0.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
-ingestr-0.10.0rc0.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.10.0rc0.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.10.0rc0.dist-info/RECORD,,
+ingestr-0.10.0rc1.dist-info/METADATA,sha256=FEWuTxIM11p3nERDqWGnj9NeaaNx4fy9bEJ4r4msuSw,7063
+ingestr-0.10.0rc1.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
+ingestr-0.10.0rc1.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.10.0rc1.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.10.0rc1.dist-info/RECORD,,