ingestr 0.3.1__tar.gz → 0.3.3__tar.gz

{ingestr-0.3.1 → ingestr-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.3.1
+Version: 0.3.3
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.3.1 → ingestr-0.3.3}/docs/supported-sources/notion.md

@@ -25,24 +25,7 @@ Once you complete the guide, you should have an API key, and the table ID to con
 ingestr ingest --source-uri 'notion://?api_key=secret_12345' --source-table 'bfeaafc0c25f40a9asdasd672a9456f3' --dest-uri duckdb:///notion.duckdb --dest-table 'notion.output'
 ```
 
-The result of this command will be a bunch of tables in the `notion.duckdb` database. The Notion integration creates a bunch of extra tables in the schema to keep track of additional information about every field in a database. You should take some time to play around with the data and understand how it's structured, and take a good look at `_dlt_parent_id` column in the tables to understand the relationships between tables.
-
-Take a look at the following Notion table:
-![an example Notion database](./images/notion_example.png)
-
-Ingesting this table using ingestr will create a bunch of new tables with quite a lot of details in them. The following query is a reconstruction of the table as it looks on Notion:
-
-```sql
-select n.text__content, s.text__content, o.properties__numerical_value__number, r.text__content
-from notion.output o
-         join notion.output__properties__name__title n on n._dlt_parent_id = o._dlt_id
-         join notion.output__properties__another_col__rich_text r on r._dlt_parent_id = o._dlt_id
-         join notion.output__properties__second_value__rich_text s on s._dlt_parent_id = o._dlt_id
-order by 1;
-```
-
-Take this as a starting point and play around with the data.
-
+The result of this command will be a table in the `notion.duckdb` database with JSON columns. 
 
 > [!CAUTION]
 > Notion does not support incremental loading, which means every time you run the command, it will copy the entire table from Notion to the destination. This can be slow for large tables.

{ingestr-0.3.1 → ingestr-0.3.3}/ingestr/main.py

@@ -1,5 +1,6 @@
 import hashlib
 from datetime import datetime
+from enum import Enum
 from typing import Optional
 
 import dlt
@@ -41,6 +42,9 @@ PARQUET_SUPPORTED_DESTINATIONS = [
     "synapse",
 ]
 
+# these sources would return a JSON for sure, which means they cannot be used with Parquet loader for BigQuery
+JSON_RETURNING_SOURCES = ["notion"]
+
 
 class SpinnerCollector(Collector):
     status: Status
@@ -80,6 +84,30 @@ class SpinnerCollector(Collector):
         self.status.stop()
 
 
+class IncrementalStrategy(str, Enum):
+    create_replace = "replace"
+    append = "append"
+    delete_insert = "delete+insert"
+    merge = "merge"
+
+
+class LoaderFileFormat(str, Enum):
+    jsonl = "jsonl"
+    parquet = "parquet"
+    insert_values = "insert_values"
+    csv = "csv"
+
+
+class SqlBackend(str, Enum):
+    sqlalchemy = "sqlalchemy"
+    pyarrow = "pyarrow"
+
+
+class Progress(str, Enum):
+    interactive = "interactive"
+    log = "log"
+
+
 @app.command()
 def ingest(
     source_uri: Annotated[
@@ -114,12 +142,12 @@ def ingest(
         ),
     ] = None,  # type: ignore
     incremental_strategy: Annotated[
-        str,
+        IncrementalStrategy,
         typer.Option(
-            help="The incremental strategy to use, must be one of 'replace', 'append', 'delete+insert', or 'merge'",
+            help="The incremental strategy to use",
             envvar="INCREMENTAL_STRATEGY",
         ),
-    ] = "replace",  # type: ignore
+    ] = IncrementalStrategy.create_replace,  # type: ignore
     interval_start: Annotated[
         Optional[datetime],
         typer.Option(
@@ -158,26 +186,26 @@ def ingest(
         ),
     ] = False,  # type: ignore
     progress: Annotated[
-        Optional[str],
+        Progress,
         typer.Option(
             help="The progress display type, must be one of 'interactive', 'log'",
             envvar="PROGRESS",
         ),
-    ] = "interactive",  # type: ignore
+    ] = Progress.interactive,  # type: ignore
     sql_backend: Annotated[
-        Optional[str],
+        SqlBackend,
         typer.Option(
-            help="The SQL backend to use, must be one of 'sqlalchemy', 'pyarrow'",
+            help="The SQL backend to use",
             envvar="SQL_BACKEND",
         ),
-    ] = "pyarrow",  # type: ignore
+    ] = SqlBackend.pyarrow,  # type: ignore
     loader_file_format: Annotated[
-        Optional[str],
+        Optional[LoaderFileFormat],
         typer.Option(
-            help="The file format to use when loading data, must be one of 'jsonl', 'parquet', 'default'",
+            help="The file format to use when loading data",
             envvar="LOADER_FILE_FORMAT",
         ),
-    ] = "default",  # type: ignore
+    ] = None,  # type: ignore
 ):
     track(
         "command_triggered",
@@ -207,15 +235,15 @@ def ingest(
         original_incremental_strategy = incremental_strategy
 
         merge_key = None
-        if incremental_strategy == "delete+insert":
+        if incremental_strategy == IncrementalStrategy.delete_insert:
             merge_key = incremental_key
-            incremental_strategy = "merge"
+            incremental_strategy = IncrementalStrategy.merge
 
         m = hashlib.sha256()
         m.update(dest_table.encode("utf-8"))
 
         progressInstance: Collector = SpinnerCollector()
-        if progress == "log":
+        if progress == Progress.log:
             progressInstance = LogCollector()
 
         pipeline = dlt.pipeline(
@@ -237,7 +265,7 @@ def ingest(
             f"[bold yellow]  Destination:[/bold yellow] {factory.destination_scheme} / {dest_table}"
         )
         print(
-            f"[bold yellow]  Incremental Strategy:[/bold yellow] {incremental_strategy}"
+            f"[bold yellow]  Incremental Strategy:[/bold yellow] {incremental_strategy.value}"
         )
         print(
             f"[bold yellow]  Incremental Key:[/bold yellow] {incremental_key if incremental_key else 'None'}"
@@ -263,19 +291,24 @@ def ingest(
             merge_key=merge_key,
             interval_start=interval_start,
             interval_end=interval_end,
-            sql_backend=sql_backend,
+            sql_backend=sql_backend.value,
         )
 
-        if original_incremental_strategy == "delete+insert":
+        if original_incremental_strategy == IncrementalStrategy.delete_insert:
             dlt_source.incremental.primary_key = ()
 
         if (
             factory.destination_scheme in PARQUET_SUPPORTED_DESTINATIONS
-            and loader_file_format == "default"
+            and loader_file_format is None
         ):
-            loader_file_format = "parquet"
-        elif loader_file_format == "default":
-            loader_file_format = "jsonl"
+            loader_file_format = LoaderFileFormat.parquet
+
+            # if the source is a JSON returning source, we cannot use Parquet loader for BigQuery
+            if (
+                factory.destination_scheme == "bigquery"
+                and factory.source_scheme in JSON_RETURNING_SOURCES
+            ):
+                loader_file_format = None
 
         run_info = pipeline.run(
             dlt_source,
@@ -283,9 +316,11 @@ def ingest(
                 uri=dest_uri,
                 table=dest_table,
             ),
-            write_disposition=incremental_strategy,  # type: ignore
+            write_disposition=incremental_strategy.value,  # type: ignore
             primary_key=(primary_key if primary_key and len(primary_key) > 0 else None),  # type: ignore
-            loader_file_format=loader_file_format,  # type: ignore
+            loader_file_format=loader_file_format.value
+            if loader_file_format is not None
+            else None,  # type: ignore
         )
 
         destination.post_load()

{ingestr-0.3.1 → ingestr-0.3.3}/ingestr/src/notion/__init__.py

@@ -9,7 +9,7 @@ from .helpers.client import NotionClient
 from .helpers.database import NotionDatabase
 
 
-@dlt.source
+@dlt.source(max_table_nesting=1)
 def notion_databases(
     database_ids: Optional[List[Dict[str, str]]] = None,
     api_key: str = dlt.secrets.value,

ingestr-0.3.3/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.3.3"

ingestr-0.3.1/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.3.1"