ingestr 0.3.1__py3-none-any.whl → 0.3.2__py3-none-any.whl

ingestr/main.py

@@ -41,6 +41,9 @@ PARQUET_SUPPORTED_DESTINATIONS = [
     "synapse",
 ]
 
+# these sources would return a JSON for sure, which means they cannot be used with Parquet loader for BigQuery
+JSON_RETURNING_SOURCES = ["notion"]
+
 
 class SpinnerCollector(Collector):
     status: Status
@@ -274,6 +277,11 @@ def ingest(
             and loader_file_format == "default"
         ):
             loader_file_format = "parquet"
+
+            # if the source is a JSON returning source, we cannot use Parquet loader for BigQuery
+            if factory.destination_scheme == 'bigquery' and factory.source_scheme in JSON_RETURNING_SOURCES:
+                loader_file_format = "jsonl"
+            
         elif loader_file_format == "default":
             loader_file_format = "jsonl"
 

ingestr/src/notion/__init__.py

@@ -9,7 +9,7 @@ from .helpers.client import NotionClient
 from .helpers.database import NotionDatabase
 
 
-@dlt.source
+@dlt.source(max_table_nesting=1)
 def notion_databases(
     database_ids: Optional[List[Dict[str, str]]] = None,
     api_key: str = dlt.secrets.value,

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.3.1"
+__version__ = "0.3.2"

{ingestr-0.3.1.dist-info → ingestr-0.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.3.1
+Version: 0.3.2
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.3.1.dist-info → ingestr-0.3.2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-ingestr/main.py,sha256=mSTj_91vPxvk-ObqD63H3L4K2b1mZ2MHMUJofqljRTY,12959
+ingestr/main.py,sha256=CLwBkFFltEaCWyZObtonQ662nIGb5S-1t4OKZ-iqgFo,13373
 ingestr/main_test.py,sha256=fXZw1qZd5CElrFSRnsI5003813LcIMGpGCMnGNFVhNI,19946
 ingestr/src/destinations.py,sha256=_PIoAU-_tDEyX_-vDOgGB5eqXoGhPwtCRApHufj1ae4,6350
 ingestr/src/destinations_test.py,sha256=rgEk8EpAntFbSOwXovC4prv3RA22mwq8pIO6sZ_rYzg,4212
@@ -6,10 +6,10 @@ ingestr/src/factory.py,sha256=DvzeYaE32pL9XpUUiO4Iaaj4ShRn9KABVeSo1E_QVg4,3114
 ingestr/src/factory_test.py,sha256=X9sFkvNByWChIcyeDt1QiIPMIzGNKb7M5A_GUE0-nnI,664
 ingestr/src/sources.py,sha256=Bl8Q1gXGqYWAZtwyF2CoaPB5CgAwoctXHwW9OZYK__8,3978
 ingestr/src/sources_test.py,sha256=t94u1lYAspxzfe-DkxVtq5vw6xrLWphipvwntrwrzqg,3930
-ingestr/src/version.py,sha256=r4xAFihOf72W9TD-lpMi6ntWSTKTP2SlzKP1ytkjRbI,22
+ingestr/src/version.py,sha256=vNiWJ14r_cw5t_7UDqDQIVZvladKFGyHH2avsLpN7Vg,22
 ingestr/src/mongodb/__init__.py,sha256=E7SDeCyYNkYZZ_RFhjCRDZUGpKtaxpPG5sFSmKJV62U,4336
 ingestr/src/mongodb/helpers.py,sha256=80vtAeNyUn1iMN0CeLrTlKqYN6I6fHF81Kd2UuE8Kns,5653
-ingestr/src/notion/__init__.py,sha256=K4TXZGysRNxeVnpdI3hx3joV-S_2Or2NA_v64cEjWnE,1813
+ingestr/src/notion/__init__.py,sha256=36wUui8finbc85ObkRMq8boMraXMUehdABN_AMe_hzA,1834
 ingestr/src/notion/settings.py,sha256=MwQVZViJtnvOegfjXYc_pJ50oUYgSRPgwqu7TvpeMOA,82
 ingestr/src/notion/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestr/src/notion/helpers/client.py,sha256=QXuudkf5Zzff98HRsCqA1g1EZWIrnfn1falPrnKg_y4,5500
@@ -21,13 +21,13 @@ ingestr/src/sql_database/schema_types.py,sha256=foGHh4iGagGLfS7nF3uGYhBjqgX0jlrj
 ingestr/src/telemetry/event.py,sha256=MpWc5tt0lSJ1pWKe9HQ11BHrcPBxSH40l4wjZi9u0tI,924
 ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
 ingestr/testdata/.gitignore,sha256=DFzYYOpqdTiT7S1HjCT-jffZSmEvFZge295_upAB0FY,13
-ingestr/testdata/test_append.db,sha256=sihcntg2HgvuzVvRQikwRRzveIHv-2q7JIGNPzQ3c2o,798720
-ingestr/testdata/test_create_replace.db,sha256=7GXhkB-ck282qq1dLjnVk0PXSzI4kxJtE_jR4UlzAL4,798720
-ingestr/testdata/test_delete_insert_with_timerange.db,sha256=rY1aCkJb-Wwv_XQUtIfGceZtFySWqgS4HK58wEM3IXM,1585152
-ingestr/testdata/test_delete_insert_without_primary_key.db,sha256=y-I8Nocvmw9HzFW2W0daloHg7gggu0hatZxZP1bg4xI,1847296
-ingestr/testdata/test_merge_with_primary_key.db,sha256=dR9cUzi7qU-aowEmxmjVZBHCnpVXnztdoIf2l24rLnA,1847296
-ingestr-0.3.1.dist-info/METADATA,sha256=n8Ue7VCewaOXazK0KAKK6IcZo6cPbH6cr2FAvBnSoGA,5309
-ingestr-0.3.1.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
-ingestr-0.3.1.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.3.1.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.3.1.dist-info/RECORD,,
+ingestr/testdata/test_append.db,sha256=3Jwyusio7tDVFtBAMZkPSdoYfCmAKms1O3QKBm9m85o,798720
+ingestr/testdata/test_create_replace.db,sha256=EQfucKIJ5SpWK0DKpISwwI2YWpoPgcieyiI_e6xtb-M,798720
+ingestr/testdata/test_delete_insert_with_timerange.db,sha256=xORZj07gvaDnQM8TgPsIx3Y1LHu9LyWP6QmkCIBBdIo,1585152
+ingestr/testdata/test_delete_insert_without_primary_key.db,sha256=fxnI0HB3Ag1aRn80t0WgaUAGmcnJsaGe26lmBWZYhCA,1847296
+ingestr/testdata/test_merge_with_primary_key.db,sha256=3p5_k-Qpt47atdAyiQ3JSk9pwqaVWq2U1LkCO-CrTCM,1847296
+ingestr-0.3.2.dist-info/METADATA,sha256=vvm_R9Tt4cWKm1aqPr9szgwSIGcL8XfoNiZZz4ZFt04,5309
+ingestr-0.3.2.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
+ingestr-0.3.2.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.3.2.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.3.2.dist-info/RECORD,,