ingestr 0.3.0__py3-none-any.whl → 0.3.2__py3-none-any.whl

ingestr/main.py

@@ -32,6 +32,18 @@ DATE_FORMATS = [
     "%Y-%m-%dT%H:%M:%S.%f%z",
 ]
 
+# https://dlthub.com/docs/dlt-ecosystem/file-formats/parquet#supported-destinations
+PARQUET_SUPPORTED_DESTINATIONS = [
+    "bigquery",
+    "duckdb",
+    "snowflake",
+    "databricks",
+    "synapse",
+]
+
+# these sources would return a JSON for sure, which means they cannot be used with Parquet loader for BigQuery
+JSON_RETURNING_SOURCES = ["notion"]
+
 
 class SpinnerCollector(Collector):
     status: Status
@@ -155,6 +167,20 @@ def ingest(
             envvar="PROGRESS",
         ),
     ] = "interactive",  # type: ignore
+    sql_backend: Annotated[
+        Optional[str],
+        typer.Option(
+            help="The SQL backend to use, must be one of 'sqlalchemy', 'pyarrow'",
+            envvar="SQL_BACKEND",
+        ),
+    ] = "pyarrow",  # type: ignore
+    loader_file_format: Annotated[
+        Optional[str],
+        typer.Option(
+            help="The file format to use when loading data, must be one of 'jsonl', 'parquet', 'default'",
+            envvar="LOADER_FILE_FORMAT",
+        ),
+    ] = "default",  # type: ignore
 ):
     track(
         "command_triggered",
@@ -240,11 +266,25 @@ def ingest(
             merge_key=merge_key,
             interval_start=interval_start,
             interval_end=interval_end,
+            sql_backend=sql_backend,
         )
 
         if original_incremental_strategy == "delete+insert":
             dlt_source.incremental.primary_key = ()
 
+        if (
+            factory.destination_scheme in PARQUET_SUPPORTED_DESTINATIONS
+            and loader_file_format == "default"
+        ):
+            loader_file_format = "parquet"
+
+            # if the source is a JSON returning source, we cannot use Parquet loader for BigQuery
+            if factory.destination_scheme == 'bigquery' and factory.source_scheme in JSON_RETURNING_SOURCES:
+                loader_file_format = "jsonl"
+            
+        elif loader_file_format == "default":
+            loader_file_format = "jsonl"
+
         run_info = pipeline.run(
             dlt_source,
             **destination.dlt_run_params(
@@ -253,6 +293,7 @@ def ingest(
             ),
             write_disposition=incremental_strategy,  # type: ignore
             primary_key=(primary_key if primary_key and len(primary_key) > 0 else None),  # type: ignore
+            loader_file_format=loader_file_format,  # type: ignore
         )
 
         destination.post_load()

ingestr/main_test.py

@@ -24,6 +24,8 @@ def invoke_ingest_command(
     merge_key=None,
     interval_start=None,
     interval_end=None,
+    sql_backend=None,
+    loader_file_format=None,
 ):
     args = [
         "ingest",
@@ -61,6 +63,14 @@ def invoke_ingest_command(
         args.append("--interval-end")
         args.append(interval_end)
 
+    if sql_backend:
+        args.append("--sql-backend")
+        args.append(sql_backend)
+
+    if loader_file_format:
+        args.append("--loader-file-format")
+        args.append(loader_file_format)
+
     result = runner.invoke(
         app,
         args,
@@ -93,7 +103,6 @@ def test_create_replace():
         "testschema.output",
     )
 
-    print(result.stdout)
     assert result.exit_code == 0
 
     res = conn.sql(
@@ -138,6 +147,7 @@ def test_append():
             "testschema_append.output",
             "append",
             "updated_at",
+            sql_backend="sqlalchemy",
         )
         assert res.exit_code == 0
 
@@ -194,6 +204,7 @@ def test_merge_with_primary_key():
             "merge",
             "updated_at",
             "id",
+            sql_backend="sqlalchemy",
         )
         assert res.exit_code == 0
         return res
@@ -351,6 +362,8 @@ def test_delete_insert_without_primary_key():
             "testschema_delete_insert.output",
             inc_strategy="delete+insert",
             inc_key="updated_at",
+            sql_backend="sqlalchemy",
+            loader_file_format="jsonl",
         )
         assert res.exit_code == 0
         return res
@@ -465,6 +478,8 @@ def test_delete_insert_with_timerange():
             inc_key="updated_at",
             interval_start=start_date,
             interval_end=end_date,
+            sql_backend="sqlalchemy",
+            loader_file_format="jsonl",
         )
         assert res.exit_code == 0
         return res

ingestr/src/notion/__init__.py

@@ -9,7 +9,7 @@ from .helpers.client import NotionClient
 from .helpers.database import NotionDatabase
 
 
-@dlt.source
+@dlt.source(max_table_nesting=1)
 def notion_databases(
     database_ids: Optional[List[Dict[str, str]]] = None,
     api_key: str = dlt.secrets.value,

ingestr/src/sources.py

@@ -41,6 +41,7 @@ class SqlSource:
             table=table_fields[-1],
             incremental=incremental,
             merge_key=kwargs.get("merge_key"),
+            backend=kwargs.get("sql_backend", "sqlalchemy"),
         )
 
         return table_instance

ingestr/src/sources_test.py

@@ -22,10 +22,11 @@ class SqlSourceTest(unittest.TestCase):
         table = "schema.table"
 
         # monkey patch the sql_table function
-        def sql_table(credentials, schema, table, incremental, merge_key):
+        def sql_table(credentials, schema, table, incremental, merge_key, backend):
             self.assertEqual(credentials, uri)
             self.assertEqual(schema, "schema")
             self.assertEqual(table, "table")
+            self.assertEqual(backend, "sqlalchemy")
             self.assertIsNone(incremental)
             self.assertIsNone(merge_key)
             return dlt.resource()
@@ -40,10 +41,11 @@ class SqlSourceTest(unittest.TestCase):
         incremental_key = "id"
 
         # monkey patch the sql_table function
-        def sql_table(credentials, schema, table, incremental, merge_key):
+        def sql_table(credentials, schema, table, incremental, merge_key, backend):
             self.assertEqual(credentials, uri)
             self.assertEqual(schema, "schema")
             self.assertEqual(table, "table")
+            self.assertEqual(backend, "sqlalchemy")
             self.assertIsInstance(incremental, dlt.sources.incremental)
             self.assertEqual(incremental.cursor_path, incremental_key)
             self.assertIsNone(merge_key)

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.3.0"
+__version__ = "0.3.2"

{ingestr-0.3.0.dist-info → ingestr-0.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.3.0
+Version: 0.3.2
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.3.0.dist-info → ingestr-0.3.2.dist-info}/RECORD

@@ -1,15 +1,15 @@
-ingestr/main.py,sha256=OIxnk6VpzQaND9rHcCZGm96rTt0IYnuHv97HsuNV2lU,11867
-ingestr/main_test.py,sha256=c8C7CUC1dJ4BrMZImjGM2yhmNONeJX_duIaz8hhtU6Y,19485
+ingestr/main.py,sha256=CLwBkFFltEaCWyZObtonQ662nIGb5S-1t4OKZ-iqgFo,13373
+ingestr/main_test.py,sha256=fXZw1qZd5CElrFSRnsI5003813LcIMGpGCMnGNFVhNI,19946
 ingestr/src/destinations.py,sha256=_PIoAU-_tDEyX_-vDOgGB5eqXoGhPwtCRApHufj1ae4,6350
 ingestr/src/destinations_test.py,sha256=rgEk8EpAntFbSOwXovC4prv3RA22mwq8pIO6sZ_rYzg,4212
 ingestr/src/factory.py,sha256=DvzeYaE32pL9XpUUiO4Iaaj4ShRn9KABVeSo1E_QVg4,3114
 ingestr/src/factory_test.py,sha256=X9sFkvNByWChIcyeDt1QiIPMIzGNKb7M5A_GUE0-nnI,664
-ingestr/src/sources.py,sha256=SKWaC8mwNArg_JVQcSJsO8w8yZs3a97V8nM84YudRJA,3917
-ingestr/src/sources_test.py,sha256=gqqnJIqblxDbipsX3jZWl0He7aMsob0YyewceY4Z11M,3808
-ingestr/src/version.py,sha256=VrXpHDu3erkzwl_WXrqINBm9xWkcyUy53IQOj042dOs,22
+ingestr/src/sources.py,sha256=Bl8Q1gXGqYWAZtwyF2CoaPB5CgAwoctXHwW9OZYK__8,3978
+ingestr/src/sources_test.py,sha256=t94u1lYAspxzfe-DkxVtq5vw6xrLWphipvwntrwrzqg,3930
+ingestr/src/version.py,sha256=vNiWJ14r_cw5t_7UDqDQIVZvladKFGyHH2avsLpN7Vg,22
 ingestr/src/mongodb/__init__.py,sha256=E7SDeCyYNkYZZ_RFhjCRDZUGpKtaxpPG5sFSmKJV62U,4336
 ingestr/src/mongodb/helpers.py,sha256=80vtAeNyUn1iMN0CeLrTlKqYN6I6fHF81Kd2UuE8Kns,5653
-ingestr/src/notion/__init__.py,sha256=K4TXZGysRNxeVnpdI3hx3joV-S_2Or2NA_v64cEjWnE,1813
+ingestr/src/notion/__init__.py,sha256=36wUui8finbc85ObkRMq8boMraXMUehdABN_AMe_hzA,1834
 ingestr/src/notion/settings.py,sha256=MwQVZViJtnvOegfjXYc_pJ50oUYgSRPgwqu7TvpeMOA,82
 ingestr/src/notion/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestr/src/notion/helpers/client.py,sha256=QXuudkf5Zzff98HRsCqA1g1EZWIrnfn1falPrnKg_y4,5500
@@ -21,13 +21,13 @@ ingestr/src/sql_database/schema_types.py,sha256=foGHh4iGagGLfS7nF3uGYhBjqgX0jlrj
 ingestr/src/telemetry/event.py,sha256=MpWc5tt0lSJ1pWKe9HQ11BHrcPBxSH40l4wjZi9u0tI,924
 ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
 ingestr/testdata/.gitignore,sha256=DFzYYOpqdTiT7S1HjCT-jffZSmEvFZge295_upAB0FY,13
-ingestr/testdata/test_append.db,sha256=YAK31Am4R2UwesMDov0qiEVW9QrvbpuSO01JV1SVmoY,798720
-ingestr/testdata/test_create_replace.db,sha256=m-Mgp-8LR82r3dJu6-NY4yDOREHCJSBOvwESdypiyNE,798720
-ingestr/testdata/test_delete_insert_with_timerange.db,sha256=5qiW6X69rCawByXbnbk8bVLDdtNGFiKDjbsNtVP7CBI,1585152
-ingestr/testdata/test_delete_insert_without_primary_key.db,sha256=WCGT3sQAL4WfJrC01kEz-A3VJncSETGeDB1wFoL-Gqc,1847296
-ingestr/testdata/test_merge_with_primary_key.db,sha256=JGvUKv6_6ArgIA5v4wUQ74_6udlmRk7eN1G_n5Oi468,1847296
-ingestr-0.3.0.dist-info/METADATA,sha256=2oj-a494cJfiAtOgdM3IFSO7kb7FuJyopiMcO0_N5RM,5309
-ingestr-0.3.0.dist-info/WHEEL,sha256=osohxoshIHTFJFVPhsi1UkZuLRGMHRXZzwEBW2ezjrc,87
-ingestr-0.3.0.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.3.0.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.3.0.dist-info/RECORD,,
+ingestr/testdata/test_append.db,sha256=3Jwyusio7tDVFtBAMZkPSdoYfCmAKms1O3QKBm9m85o,798720
+ingestr/testdata/test_create_replace.db,sha256=EQfucKIJ5SpWK0DKpISwwI2YWpoPgcieyiI_e6xtb-M,798720
+ingestr/testdata/test_delete_insert_with_timerange.db,sha256=xORZj07gvaDnQM8TgPsIx3Y1LHu9LyWP6QmkCIBBdIo,1585152
+ingestr/testdata/test_delete_insert_without_primary_key.db,sha256=fxnI0HB3Ag1aRn80t0WgaUAGmcnJsaGe26lmBWZYhCA,1847296
+ingestr/testdata/test_merge_with_primary_key.db,sha256=3p5_k-Qpt47atdAyiQ3JSk9pwqaVWq2U1LkCO-CrTCM,1847296
+ingestr-0.3.2.dist-info/METADATA,sha256=vvm_R9Tt4cWKm1aqPr9szgwSIGcL8XfoNiZZz4ZFt04,5309
+ingestr-0.3.2.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
+ingestr-0.3.2.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.3.2.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.3.2.dist-info/RECORD,,

{ingestr-0.3.0.dist-info → ingestr-0.3.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.24.1
+Generator: hatchling 1.24.2
 Root-Is-Purelib: true
 Tag: py3-none-any