ingestr 0.3.0__tar.gz → 0.3.1__tar.gz

{ingestr-0.3.0 → ingestr-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.3.0
+Version: 0.3.1
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.3.0 → ingestr-0.3.1}/ingestr/main.py

@@ -32,6 +32,15 @@ DATE_FORMATS = [
     "%Y-%m-%dT%H:%M:%S.%f%z",
 ]
 
+# https://dlthub.com/docs/dlt-ecosystem/file-formats/parquet#supported-destinations
+PARQUET_SUPPORTED_DESTINATIONS = [
+    "bigquery",
+    "duckdb",
+    "snowflake",
+    "databricks",
+    "synapse",
+]
+
 
 class SpinnerCollector(Collector):
     status: Status
@@ -155,6 +164,20 @@ def ingest(
             envvar="PROGRESS",
         ),
     ] = "interactive",  # type: ignore
+    sql_backend: Annotated[
+        Optional[str],
+        typer.Option(
+            help="The SQL backend to use, must be one of 'sqlalchemy', 'pyarrow'",
+            envvar="SQL_BACKEND",
+        ),
+    ] = "pyarrow",  # type: ignore
+    loader_file_format: Annotated[
+        Optional[str],
+        typer.Option(
+            help="The file format to use when loading data, must be one of 'jsonl', 'parquet', 'default'",
+            envvar="LOADER_FILE_FORMAT",
+        ),
+    ] = "default",  # type: ignore
 ):
     track(
         "command_triggered",
@@ -240,11 +263,20 @@ def ingest(
             merge_key=merge_key,
             interval_start=interval_start,
             interval_end=interval_end,
+            sql_backend=sql_backend,
         )
 
         if original_incremental_strategy == "delete+insert":
             dlt_source.incremental.primary_key = ()
 
+        if (
+            factory.destination_scheme in PARQUET_SUPPORTED_DESTINATIONS
+            and loader_file_format == "default"
+        ):
+            loader_file_format = "parquet"
+        elif loader_file_format == "default":
+            loader_file_format = "jsonl"
+
         run_info = pipeline.run(
             dlt_source,
             **destination.dlt_run_params(
@@ -253,6 +285,7 @@ def ingest(
             ),
             write_disposition=incremental_strategy,  # type: ignore
             primary_key=(primary_key if primary_key and len(primary_key) > 0 else None),  # type: ignore
+            loader_file_format=loader_file_format,  # type: ignore
         )
 
         destination.post_load()

{ingestr-0.3.0 → ingestr-0.3.1}/ingestr/main_test.py

@@ -24,6 +24,8 @@ def invoke_ingest_command(
     merge_key=None,
     interval_start=None,
     interval_end=None,
+    sql_backend=None,
+    loader_file_format=None,
 ):
     args = [
         "ingest",
@@ -61,6 +63,14 @@ def invoke_ingest_command(
         args.append("--interval-end")
         args.append(interval_end)
 
+    if sql_backend:
+        args.append("--sql-backend")
+        args.append(sql_backend)
+
+    if loader_file_format:
+        args.append("--loader-file-format")
+        args.append(loader_file_format)
+
     result = runner.invoke(
         app,
         args,
@@ -93,7 +103,6 @@ def test_create_replace():
         "testschema.output",
     )
 
-    print(result.stdout)
     assert result.exit_code == 0
 
     res = conn.sql(
@@ -138,6 +147,7 @@ def test_append():
             "testschema_append.output",
             "append",
             "updated_at",
+            sql_backend="sqlalchemy",
         )
         assert res.exit_code == 0
 
@@ -194,6 +204,7 @@ def test_merge_with_primary_key():
             "merge",
             "updated_at",
             "id",
+            sql_backend="sqlalchemy",
         )
         assert res.exit_code == 0
         return res
@@ -351,6 +362,8 @@ def test_delete_insert_without_primary_key():
             "testschema_delete_insert.output",
             inc_strategy="delete+insert",
             inc_key="updated_at",
+            sql_backend="sqlalchemy",
+            loader_file_format="jsonl",
         )
         assert res.exit_code == 0
         return res
@@ -465,6 +478,8 @@ def test_delete_insert_with_timerange():
             inc_key="updated_at",
             interval_start=start_date,
             interval_end=end_date,
+            sql_backend="sqlalchemy",
+            loader_file_format="jsonl",
         )
         assert res.exit_code == 0
         return res

{ingestr-0.3.0 → ingestr-0.3.1}/ingestr/src/sources.py

@@ -41,6 +41,7 @@ class SqlSource:
             table=table_fields[-1],
             incremental=incremental,
             merge_key=kwargs.get("merge_key"),
+            backend=kwargs.get("sql_backend", "sqlalchemy"),
         )
 
         return table_instance

{ingestr-0.3.0 → ingestr-0.3.1}/ingestr/src/sources_test.py

@@ -22,10 +22,11 @@ class SqlSourceTest(unittest.TestCase):
         table = "schema.table"
 
         # monkey patch the sql_table function
-        def sql_table(credentials, schema, table, incremental, merge_key):
+        def sql_table(credentials, schema, table, incremental, merge_key, backend):
             self.assertEqual(credentials, uri)
             self.assertEqual(schema, "schema")
             self.assertEqual(table, "table")
+            self.assertEqual(backend, "sqlalchemy")
             self.assertIsNone(incremental)
             self.assertIsNone(merge_key)
             return dlt.resource()
@@ -40,10 +41,11 @@ class SqlSourceTest(unittest.TestCase):
         incremental_key = "id"
 
         # monkey patch the sql_table function
-        def sql_table(credentials, schema, table, incremental, merge_key):
+        def sql_table(credentials, schema, table, incremental, merge_key, backend):
             self.assertEqual(credentials, uri)
             self.assertEqual(schema, "schema")
             self.assertEqual(table, "table")
+            self.assertEqual(backend, "sqlalchemy")
             self.assertIsInstance(incremental, dlt.sources.incremental)
             self.assertEqual(incremental.cursor_path, incremental_key)
             self.assertIsNone(merge_key)

ingestr-0.3.1/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.3.1"

ingestr-0.3.0/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.3.0"