ingestr 0.3.2__tar.gz → 0.4.0__tar.gz

{ingestr-0.3.2 → ingestr-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.3.2
+Version: 0.4.0
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -31,6 +31,7 @@ Requires-Dist: rich==13.7.1
 Requires-Dist: rudder-sdk-python==2.1.0
 Requires-Dist: snowflake-sqlalchemy==1.5.3
 Requires-Dist: sqlalchemy-bigquery==1.11.0
+Requires-Dist: sqlalchemy-hana==2.0.0
 Requires-Dist: sqlalchemy-redshift==0.8.14
 Requires-Dist: sqlalchemy2-stubs==0.0.2a38
 Requires-Dist: sqlalchemy==1.4.52
@@ -150,6 +151,11 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
         <td>✅</td>
         <td>❌</td>
     </tr>
+    <tr>
+        <td>SAP Hana</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
     <tr>
         <td>SQLite</td>
         <td>✅</td>

{ingestr-0.3.2 → ingestr-0.4.0}/README.md

@@ -110,6 +110,11 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
         <td>✅</td>
         <td>❌</td>
     </tr>
+    <tr>
+        <td>SAP Hana</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
     <tr>
         <td>SQLite</td>
         <td>✅</td>

{ingestr-0.3.2 → ingestr-0.4.0}/docs/.vitepress/config.mjs

@@ -58,6 +58,7 @@ export default defineConfig({
               { text: "MySQL", link: "/supported-sources/mysql.md" },
               { text: "Oracle", link: "/supported-sources/oracle.md" },
               { text: "Postgres", link: "/supported-sources/postgres.md" },
+              { text: "SAP Hana", link: "/supported-sources/sap-hana.md" },
               { text: "Snowflake", link: "/supported-sources/snowflake.md" },
               { text: "SQLite", link: "/supported-sources/sqlite.md" },
             ],

{ingestr-0.3.2 → ingestr-0.4.0}/docs/supported-sources/overview.md

@@ -61,6 +61,11 @@ ingestr supports the following sources and destinations:
         <td>✅</td>
         <td>❌</td>
     </tr>
+    <tr>
+        <td>SAP Hana</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
     <tr>
         <td>SQLite</td>
         <td>✅</td>

ingestr-0.4.0/docs/supported-sources/sap-hana.md

@@ -0,0 +1,20 @@
+# SAP HANA
+SAP HANA is an in-memory, column-oriented, relational database management system.
+
+ingestr supports SAP HANA as both a source and destination. It uses the [SQLAlchemy connector for SAP Hana](https://github.com/SAP/sqlalchemy-hana/), so the connection options there would all be valid.
+
+## URI Format
+The URI format for SAP HANA is as follows:
+
+```plaintext
+hana://user:password@host:port/dbname
+```
+
+URI parameters:
+- `user`: the username to connect to the database
+- `password`: the password for the user
+- `host`: the host address of the database server
+- `port`: the port number the database server is listening on (default is 30015)
+- `dbname`: the name of the database to connect to
+
+The URI structure can be used both for sources and destinations. More details about SAP HANA’s JDBC and ODBC interfaces can be found [here](https://github.com/SAP/sqlalchemy-hana/).

{ingestr-0.3.2 → ingestr-0.4.0}/ingestr/main.py

@@ -1,5 +1,6 @@
 import hashlib
 from datetime import datetime
+from enum import Enum
 from typing import Optional
 
 import dlt
@@ -83,6 +84,30 @@ class SpinnerCollector(Collector):
         self.status.stop()
 
 
+class IncrementalStrategy(str, Enum):
+    create_replace = "replace"
+    append = "append"
+    delete_insert = "delete+insert"
+    merge = "merge"
+
+
+class LoaderFileFormat(str, Enum):
+    jsonl = "jsonl"
+    parquet = "parquet"
+    insert_values = "insert_values"
+    csv = "csv"
+
+
+class SqlBackend(str, Enum):
+    sqlalchemy = "sqlalchemy"
+    pyarrow = "pyarrow"
+
+
+class Progress(str, Enum):
+    interactive = "interactive"
+    log = "log"
+
+
 @app.command()
 def ingest(
     source_uri: Annotated[
@@ -117,12 +142,12 @@ def ingest(
         ),
     ] = None,  # type: ignore
     incremental_strategy: Annotated[
-        str,
+        IncrementalStrategy,
         typer.Option(
-            help="The incremental strategy to use, must be one of 'replace', 'append', 'delete+insert', or 'merge'",
+            help="The incremental strategy to use",
             envvar="INCREMENTAL_STRATEGY",
         ),
-    ] = "replace",  # type: ignore
+    ] = IncrementalStrategy.create_replace,  # type: ignore
     interval_start: Annotated[
         Optional[datetime],
         typer.Option(
@@ -161,26 +186,26 @@ def ingest(
         ),
     ] = False,  # type: ignore
     progress: Annotated[
-        Optional[str],
+        Progress,
         typer.Option(
             help="The progress display type, must be one of 'interactive', 'log'",
             envvar="PROGRESS",
         ),
-    ] = "interactive",  # type: ignore
+    ] = Progress.interactive,  # type: ignore
     sql_backend: Annotated[
-        Optional[str],
+        SqlBackend,
         typer.Option(
-            help="The SQL backend to use, must be one of 'sqlalchemy', 'pyarrow'",
+            help="The SQL backend to use",
             envvar="SQL_BACKEND",
         ),
-    ] = "pyarrow",  # type: ignore
+    ] = SqlBackend.pyarrow,  # type: ignore
     loader_file_format: Annotated[
-        Optional[str],
+        Optional[LoaderFileFormat],
         typer.Option(
-            help="The file format to use when loading data, must be one of 'jsonl', 'parquet', 'default'",
+            help="The file format to use when loading data",
             envvar="LOADER_FILE_FORMAT",
         ),
-    ] = "default",  # type: ignore
+    ] = None,  # type: ignore
 ):
     track(
         "command_triggered",
@@ -210,15 +235,15 @@ def ingest(
         original_incremental_strategy = incremental_strategy
 
         merge_key = None
-        if incremental_strategy == "delete+insert":
+        if incremental_strategy == IncrementalStrategy.delete_insert:
             merge_key = incremental_key
-            incremental_strategy = "merge"
+            incremental_strategy = IncrementalStrategy.merge
 
         m = hashlib.sha256()
         m.update(dest_table.encode("utf-8"))
 
         progressInstance: Collector = SpinnerCollector()
-        if progress == "log":
+        if progress == Progress.log:
             progressInstance = LogCollector()
 
         pipeline = dlt.pipeline(
@@ -240,7 +265,7 @@ def ingest(
             f"[bold yellow]  Destination:[/bold yellow] {factory.destination_scheme} / {dest_table}"
         )
         print(
-            f"[bold yellow]  Incremental Strategy:[/bold yellow] {incremental_strategy}"
+            f"[bold yellow]  Incremental Strategy:[/bold yellow] {incremental_strategy.value}"
         )
         print(
             f"[bold yellow]  Incremental Key:[/bold yellow] {incremental_key if incremental_key else 'None'}"
@@ -266,24 +291,24 @@ def ingest(
             merge_key=merge_key,
             interval_start=interval_start,
             interval_end=interval_end,
-            sql_backend=sql_backend,
+            sql_backend=sql_backend.value,
         )
 
-        if original_incremental_strategy == "delete+insert":
+        if original_incremental_strategy == IncrementalStrategy.delete_insert:
             dlt_source.incremental.primary_key = ()
 
         if (
             factory.destination_scheme in PARQUET_SUPPORTED_DESTINATIONS
-            and loader_file_format == "default"
+            and loader_file_format is None
         ):
-            loader_file_format = "parquet"
+            loader_file_format = LoaderFileFormat.parquet
 
             # if the source is a JSON returning source, we cannot use Parquet loader for BigQuery
-            if factory.destination_scheme == 'bigquery' and factory.source_scheme in JSON_RETURNING_SOURCES:
-                loader_file_format = "jsonl"
-            
-        elif loader_file_format == "default":
-            loader_file_format = "jsonl"
+            if (
+                factory.destination_scheme == "bigquery"
+                and factory.source_scheme in JSON_RETURNING_SOURCES
+            ):
+                loader_file_format = None
 
         run_info = pipeline.run(
             dlt_source,
@@ -291,9 +316,11 @@ def ingest(
                 uri=dest_uri,
                 table=dest_table,
             ),
-            write_disposition=incremental_strategy,  # type: ignore
+            write_disposition=incremental_strategy.value,  # type: ignore
             primary_key=(primary_key if primary_key and len(primary_key) > 0 else None),  # type: ignore
-            loader_file_format=loader_file_format,  # type: ignore
+            loader_file_format=loader_file_format.value
+            if loader_file_format is not None
+            else None,  # type: ignore
         )
 
         destination.post_load()

{ingestr-0.3.2 → ingestr-0.4.0}/ingestr/src/factory.py

@@ -30,6 +30,7 @@ SQL_SOURCE_SCHEMES = [
     "sqlite",
     "oracle",
     "oracle+cx_oracle",
+    "hana",
 ]
 
 

ingestr-0.4.0/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.4.0"

{ingestr-0.3.2 → ingestr-0.4.0}/requirements.txt

@@ -19,4 +19,5 @@ sqlalchemy-redshift==0.8.14
 SQLAlchemy==1.4.52
 sqlalchemy2-stubs==0.0.2a38
 tqdm==4.66.2
-typer==0.12.3
+typer==0.12.3
+sqlalchemy-hana==2.0.0

ingestr-0.3.2/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.3.2"