ingestr 0.7.0__py3-none-any.whl → 0.7.3__py3-none-any.whl

ingestr/main.py

@@ -104,6 +104,7 @@ class LoaderFileFormat(str, Enum):
 class SqlBackend(str, Enum):
     sqlalchemy = "sqlalchemy"
     pyarrow = "pyarrow"
+    connectorx = "connectorx"
 
 
 class Progress(str, Enum):
@@ -111,6 +112,11 @@ class Progress(str, Enum):
     log = "log"
 
 
+class SchemaNaming(str, Enum):
+    default = "default"
+    direct = "direct"
+
+
 @app.command()
 def ingest(
     source_uri: Annotated[
@@ -223,6 +229,13 @@ def ingest(
             envvar="LOADER_FILE_SIZE",
         ),
     ] = 100000,  # type: ignore
+    schema_naming: Annotated[
+        SchemaNaming,
+        typer.Option(
+            help="The naming convention to use when moving the tables from source to destination. The default behavior is explained here: https://dlthub.com/docs/general-usage/schema#naming-convention",
+            envvar="SCHEMA_NAMING",
+        ),
+    ] = SchemaNaming.default,  # type: ignore
 ):
     track(
         "command_triggered",
@@ -231,9 +244,9 @@ def ingest(
         },
     )
 
-    dlt.config["normalize.parquet_normalizer.add_dlt_load_id"] = True
-    dlt.config["normalize.parquet_normalizer.add_dlt_id"] = True
     dlt.config["data_writer.file_max_items"] = loader_file_size
+    if schema_naming != SchemaNaming.default:
+        dlt.config["schema.naming"] = schema_naming.value
 
     try:
         if not dest_table:
@@ -265,7 +278,7 @@ def ingest(
 
         progressInstance: Collector = SpinnerCollector()
         if progress == Progress.log:
-            progressInstance = LogCollector()
+            progressInstance = LogCollector(dump_system_stats=False)
 
         pipeline = dlt.pipeline(
             pipeline_name=m.hexdigest(),

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.7.0"
+__version__ = "0.7.3"

{ingestr-0.7.0.dist-info → ingestr-0.7.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.7.0
+Version: 0.7.3
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -14,6 +14,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Database
 Requires-Python: >=3.9
+Requires-Dist: connectorx==0.3.3
 Requires-Dist: cx-oracle==8.3.0
 Requires-Dist: databricks-sql-connector==2.9.3
 Requires-Dist: dlt==0.5.1

{ingestr-0.7.0.dist-info → ingestr-0.7.3.dist-info}/RECORD

@@ -1,9 +1,9 @@
-ingestr/main.py,sha256=JYgh3rZSO9n_Ko2D_9BLaF_cGQHrLSywK1WH6XioefQ,15961
+ingestr/main.py,sha256=j0pscsPbeJ9oYJiTCvymneZwg4Lc7KaR3GAMX0GG4To,16432
 ingestr/src/destinations.py,sha256=2SfPMjtTelPmzQmc3zNs8xGcKIPuGn_hoZFIBUuhjXI,6338
 ingestr/src/factory.py,sha256=XuT_8LvWd7gBxOjoD_NiG-jtPvHNQ9nqOeoCJzhRb6Y,3630
 ingestr/src/sources.py,sha256=QbSvECvGbHJKOpE9_dbq11343pA5ajsS9BPPPab1ivw,10007
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
-ingestr/src/version.py,sha256=RaANGbRu5e-vehwXI1-Qe2ggPPfs1TQaZj072JdbLk4,22
+ingestr/src/version.py,sha256=G0rwkDLSytonr7idr8ma7KaTUnRCn3-Ripum70RSeh0,22
 ingestr/src/google_sheets/README.md,sha256=wFQhvmGpRA38Ba2N_WIax6duyD4c7c_pwvvprRfQDnw,5470
 ingestr/src/google_sheets/__init__.py,sha256=5qlX-6ilx5MW7klC7B_0jGSxloQSLkSESTh4nlY3Aos,6643
 ingestr/src/google_sheets/helpers/__init__.py,sha256=5hXZrZK8cMO3UOuL-s4OKOpdACdihQD0hYYlSEu-iQ8,35
@@ -37,8 +37,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.7.0.dist-info/METADATA,sha256=MmQ_futv2ZZbVg4hdbCErSAgs2AobplIUr4vMErTXEI,5829
-ingestr-0.7.0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-ingestr-0.7.0.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.7.0.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.7.0.dist-info/RECORD,,
+ingestr-0.7.3.dist-info/METADATA,sha256=pLZ95a9c35K2GBcbYea9KdGBxd2QqV1R0-NXJI0QyVg,5862
+ingestr-0.7.3.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+ingestr-0.7.3.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.7.3.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.7.3.dist-info/RECORD,,