ingestr 0.3.2__py3-none-any.whl → 0.4.0__py3-none-any.whl

ingestr/main.py

@@ -1,5 +1,6 @@
 import hashlib
 from datetime import datetime
+from enum import Enum
 from typing import Optional
 
 import dlt
@@ -83,6 +84,30 @@ class SpinnerCollector(Collector):
         self.status.stop()
 
 
+class IncrementalStrategy(str, Enum):
+    create_replace = "replace"
+    append = "append"
+    delete_insert = "delete+insert"
+    merge = "merge"
+
+
+class LoaderFileFormat(str, Enum):
+    jsonl = "jsonl"
+    parquet = "parquet"
+    insert_values = "insert_values"
+    csv = "csv"
+
+
+class SqlBackend(str, Enum):
+    sqlalchemy = "sqlalchemy"
+    pyarrow = "pyarrow"
+
+
+class Progress(str, Enum):
+    interactive = "interactive"
+    log = "log"
+
+
 @app.command()
 def ingest(
     source_uri: Annotated[
@@ -117,12 +142,12 @@ def ingest(
         ),
     ] = None,  # type: ignore
     incremental_strategy: Annotated[
-        str,
+        IncrementalStrategy,
         typer.Option(
-            help="The incremental strategy to use, must be one of 'replace', 'append', 'delete+insert', or 'merge'",
+            help="The incremental strategy to use",
             envvar="INCREMENTAL_STRATEGY",
         ),
-    ] = "replace",  # type: ignore
+    ] = IncrementalStrategy.create_replace,  # type: ignore
     interval_start: Annotated[
         Optional[datetime],
         typer.Option(
@@ -161,26 +186,26 @@ def ingest(
         ),
     ] = False,  # type: ignore
     progress: Annotated[
-        Optional[str],
+        Progress,
         typer.Option(
             help="The progress display type, must be one of 'interactive', 'log'",
             envvar="PROGRESS",
         ),
-    ] = "interactive",  # type: ignore
+    ] = Progress.interactive,  # type: ignore
     sql_backend: Annotated[
-        Optional[str],
+        SqlBackend,
         typer.Option(
-            help="The SQL backend to use, must be one of 'sqlalchemy', 'pyarrow'",
+            help="The SQL backend to use",
             envvar="SQL_BACKEND",
         ),
-    ] = "pyarrow",  # type: ignore
+    ] = SqlBackend.pyarrow,  # type: ignore
     loader_file_format: Annotated[
-        Optional[str],
+        Optional[LoaderFileFormat],
         typer.Option(
-            help="The file format to use when loading data, must be one of 'jsonl', 'parquet', 'default'",
+            help="The file format to use when loading data",
             envvar="LOADER_FILE_FORMAT",
         ),
-    ] = "default",  # type: ignore
+    ] = None,  # type: ignore
 ):
     track(
         "command_triggered",
@@ -210,15 +235,15 @@ def ingest(
         original_incremental_strategy = incremental_strategy
 
         merge_key = None
-        if incremental_strategy == "delete+insert":
+        if incremental_strategy == IncrementalStrategy.delete_insert:
             merge_key = incremental_key
-            incremental_strategy = "merge"
+            incremental_strategy = IncrementalStrategy.merge
 
         m = hashlib.sha256()
         m.update(dest_table.encode("utf-8"))
 
         progressInstance: Collector = SpinnerCollector()
-        if progress == "log":
+        if progress == Progress.log:
             progressInstance = LogCollector()
 
         pipeline = dlt.pipeline(
@@ -240,7 +265,7 @@ def ingest(
             f"[bold yellow]  Destination:[/bold yellow] {factory.destination_scheme} / {dest_table}"
         )
         print(
-            f"[bold yellow]  Incremental Strategy:[/bold yellow] {incremental_strategy}"
+            f"[bold yellow]  Incremental Strategy:[/bold yellow] {incremental_strategy.value}"
         )
         print(
             f"[bold yellow]  Incremental Key:[/bold yellow] {incremental_key if incremental_key else 'None'}"
@@ -266,24 +291,24 @@ def ingest(
             merge_key=merge_key,
             interval_start=interval_start,
             interval_end=interval_end,
-            sql_backend=sql_backend,
+            sql_backend=sql_backend.value,
         )
 
-        if original_incremental_strategy == "delete+insert":
+        if original_incremental_strategy == IncrementalStrategy.delete_insert:
             dlt_source.incremental.primary_key = ()
 
         if (
             factory.destination_scheme in PARQUET_SUPPORTED_DESTINATIONS
-            and loader_file_format == "default"
+            and loader_file_format is None
         ):
-            loader_file_format = "parquet"
+            loader_file_format = LoaderFileFormat.parquet
 
             # if the source is a JSON returning source, we cannot use Parquet loader for BigQuery
-            if factory.destination_scheme == 'bigquery' and factory.source_scheme in JSON_RETURNING_SOURCES:
-                loader_file_format = "jsonl"
-            
-        elif loader_file_format == "default":
-            loader_file_format = "jsonl"
+            if (
+                factory.destination_scheme == "bigquery"
+                and factory.source_scheme in JSON_RETURNING_SOURCES
+            ):
+                loader_file_format = None
 
         run_info = pipeline.run(
             dlt_source,
@@ -291,9 +316,11 @@ def ingest(
                 uri=dest_uri,
                 table=dest_table,
             ),
-            write_disposition=incremental_strategy,  # type: ignore
+            write_disposition=incremental_strategy.value,  # type: ignore
             primary_key=(primary_key if primary_key and len(primary_key) > 0 else None),  # type: ignore
-            loader_file_format=loader_file_format,  # type: ignore
+            loader_file_format=loader_file_format.value
+            if loader_file_format is not None
+            else None,  # type: ignore
         )
 
         destination.post_load()

ingestr/src/factory.py

@@ -30,6 +30,7 @@ SQL_SOURCE_SCHEMES = [
     "sqlite",
     "oracle",
     "oracle+cx_oracle",
+    "hana",
 ]
 
 

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.3.2"
+__version__ = "0.4.0"

{ingestr-0.3.2.dist-info → ingestr-0.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.3.2
+Version: 0.4.0
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -31,6 +31,7 @@ Requires-Dist: rich==13.7.1
 Requires-Dist: rudder-sdk-python==2.1.0
 Requires-Dist: snowflake-sqlalchemy==1.5.3
 Requires-Dist: sqlalchemy-bigquery==1.11.0
+Requires-Dist: sqlalchemy-hana==2.0.0
 Requires-Dist: sqlalchemy-redshift==0.8.14
 Requires-Dist: sqlalchemy2-stubs==0.0.2a38
 Requires-Dist: sqlalchemy==1.4.52
@@ -150,6 +151,11 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
         <td>✅</td>
         <td>❌</td>
     </tr>
+    <tr>
+        <td>SAP Hana</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
     <tr>
         <td>SQLite</td>
         <td>✅</td>

{ingestr-0.3.2.dist-info → ingestr-0.4.0.dist-info}/RECORD

@@ -1,12 +1,12 @@
-ingestr/main.py,sha256=CLwBkFFltEaCWyZObtonQ662nIGb5S-1t4OKZ-iqgFo,13373
+ingestr/main.py,sha256=tXw5VRr9yyGTvd_Wp5HBdSlTt3oe61vcFvnsyljFd7g,13853
 ingestr/main_test.py,sha256=fXZw1qZd5CElrFSRnsI5003813LcIMGpGCMnGNFVhNI,19946
 ingestr/src/destinations.py,sha256=_PIoAU-_tDEyX_-vDOgGB5eqXoGhPwtCRApHufj1ae4,6350
 ingestr/src/destinations_test.py,sha256=rgEk8EpAntFbSOwXovC4prv3RA22mwq8pIO6sZ_rYzg,4212
-ingestr/src/factory.py,sha256=DvzeYaE32pL9XpUUiO4Iaaj4ShRn9KABVeSo1E_QVg4,3114
+ingestr/src/factory.py,sha256=RYdl1etpU0Q7UDzcaCjKriaUvDiMpoYJZqATQW2sums,3126
 ingestr/src/factory_test.py,sha256=X9sFkvNByWChIcyeDt1QiIPMIzGNKb7M5A_GUE0-nnI,664
 ingestr/src/sources.py,sha256=Bl8Q1gXGqYWAZtwyF2CoaPB5CgAwoctXHwW9OZYK__8,3978
 ingestr/src/sources_test.py,sha256=t94u1lYAspxzfe-DkxVtq5vw6xrLWphipvwntrwrzqg,3930
-ingestr/src/version.py,sha256=vNiWJ14r_cw5t_7UDqDQIVZvladKFGyHH2avsLpN7Vg,22
+ingestr/src/version.py,sha256=42STGor_9nKYXumfeV5tiyD_M8VdcddX7CEexmibPBk,22
 ingestr/src/mongodb/__init__.py,sha256=E7SDeCyYNkYZZ_RFhjCRDZUGpKtaxpPG5sFSmKJV62U,4336
 ingestr/src/mongodb/helpers.py,sha256=80vtAeNyUn1iMN0CeLrTlKqYN6I6fHF81Kd2UuE8Kns,5653
 ingestr/src/notion/__init__.py,sha256=36wUui8finbc85ObkRMq8boMraXMUehdABN_AMe_hzA,1834
@@ -21,13 +21,13 @@ ingestr/src/sql_database/schema_types.py,sha256=foGHh4iGagGLfS7nF3uGYhBjqgX0jlrj
 ingestr/src/telemetry/event.py,sha256=MpWc5tt0lSJ1pWKe9HQ11BHrcPBxSH40l4wjZi9u0tI,924
 ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
 ingestr/testdata/.gitignore,sha256=DFzYYOpqdTiT7S1HjCT-jffZSmEvFZge295_upAB0FY,13
-ingestr/testdata/test_append.db,sha256=3Jwyusio7tDVFtBAMZkPSdoYfCmAKms1O3QKBm9m85o,798720
-ingestr/testdata/test_create_replace.db,sha256=EQfucKIJ5SpWK0DKpISwwI2YWpoPgcieyiI_e6xtb-M,798720
-ingestr/testdata/test_delete_insert_with_timerange.db,sha256=xORZj07gvaDnQM8TgPsIx3Y1LHu9LyWP6QmkCIBBdIo,1585152
-ingestr/testdata/test_delete_insert_without_primary_key.db,sha256=fxnI0HB3Ag1aRn80t0WgaUAGmcnJsaGe26lmBWZYhCA,1847296
-ingestr/testdata/test_merge_with_primary_key.db,sha256=3p5_k-Qpt47atdAyiQ3JSk9pwqaVWq2U1LkCO-CrTCM,1847296
-ingestr-0.3.2.dist-info/METADATA,sha256=vvm_R9Tt4cWKm1aqPr9szgwSIGcL8XfoNiZZz4ZFt04,5309
-ingestr-0.3.2.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
-ingestr-0.3.2.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.3.2.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.3.2.dist-info/RECORD,,
+ingestr/testdata/test_append.db,sha256=rJUEAoJMQL5G1WETOHPcWt-ircGQJDg6NMDxMloux8g,798720
+ingestr/testdata/test_create_replace.db,sha256=wwbu-ByA_JNKcRzgKZNGiXmSjrD1kaHcHixmvYht5Tc,798720
+ingestr/testdata/test_delete_insert_with_timerange.db,sha256=-ITVU-6b2XeSBsnckjfziGGVnWnGobzXgU78ikzHw4g,1585152
+ingestr/testdata/test_delete_insert_without_primary_key.db,sha256=KkcYdlKwB-pd9_N1bo6b2MuhLBD5s77muFGvAWarMjk,1847296
+ingestr/testdata/test_merge_with_primary_key.db,sha256=Jc94_5H4q3liUDroyWQExrWT_3lSpWFoz6vne_s6KHI,1847296
+ingestr-0.4.0.dist-info/METADATA,sha256=BhqFBbVuQdkJ4mm706Pe55WNHBLGPH6n_x7QMrEqsDQ,5434
+ingestr-0.4.0.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
+ingestr-0.4.0.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.4.0.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.4.0.dist-info/RECORD,,