ingestr 0.3.0__tar.gz → 0.3.2__tar.gz

{ingestr-0.3.0 → ingestr-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.3.0
+Version: 0.3.2
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.3.0 → ingestr-0.3.2}/docs/supported-sources/notion.md

@@ -25,24 +25,7 @@ Once you complete the guide, you should have an API key, and the table ID to con
 ingestr ingest --source-uri 'notion://?api_key=secret_12345' --source-table 'bfeaafc0c25f40a9asdasd672a9456f3' --dest-uri duckdb:///notion.duckdb --dest-table 'notion.output'
 ```
 
-The result of this command will be a bunch of tables in the `notion.duckdb` database. The Notion integration creates a bunch of extra tables in the schema to keep track of additional information about every field in a database. You should take some time to play around with the data and understand how it's structured, and take a good look at `_dlt_parent_id` column in the tables to understand the relationships between tables.
-
-Take a look at the following Notion table:
-![an example Notion database](./images/notion_example.png)
-
-Ingesting this table using ingestr will create a bunch of new tables with quite a lot of details in them. The following query is a reconstruction of the table as it looks on Notion:
-
-```sql
-select n.text__content, s.text__content, o.properties__numerical_value__number, r.text__content
-from notion.output o
-         join notion.output__properties__name__title n on n._dlt_parent_id = o._dlt_id
-         join notion.output__properties__another_col__rich_text r on r._dlt_parent_id = o._dlt_id
-         join notion.output__properties__second_value__rich_text s on s._dlt_parent_id = o._dlt_id
-order by 1;
-```
-
-Take this as a starting point and play around with the data.
-
+The result of this command will be a table in the `notion.duckdb` database with JSON columns. 
 
 > [!CAUTION]
 > Notion does not support incremental loading, which means every time you run the command, it will copy the entire table from Notion to the destination. This can be slow for large tables.

{ingestr-0.3.0 → ingestr-0.3.2}/ingestr/main.py

@@ -32,6 +32,18 @@ DATE_FORMATS = [
     "%Y-%m-%dT%H:%M:%S.%f%z",
 ]
 
+# https://dlthub.com/docs/dlt-ecosystem/file-formats/parquet#supported-destinations
+PARQUET_SUPPORTED_DESTINATIONS = [
+    "bigquery",
+    "duckdb",
+    "snowflake",
+    "databricks",
+    "synapse",
+]
+
+# these sources would return a JSON for sure, which means they cannot be used with Parquet loader for BigQuery
+JSON_RETURNING_SOURCES = ["notion"]
+
 
 class SpinnerCollector(Collector):
     status: Status
@@ -155,6 +167,20 @@ def ingest(
             envvar="PROGRESS",
         ),
     ] = "interactive",  # type: ignore
+    sql_backend: Annotated[
+        Optional[str],
+        typer.Option(
+            help="The SQL backend to use, must be one of 'sqlalchemy', 'pyarrow'",
+            envvar="SQL_BACKEND",
+        ),
+    ] = "pyarrow",  # type: ignore
+    loader_file_format: Annotated[
+        Optional[str],
+        typer.Option(
+            help="The file format to use when loading data, must be one of 'jsonl', 'parquet', 'default'",
+            envvar="LOADER_FILE_FORMAT",
+        ),
+    ] = "default",  # type: ignore
 ):
     track(
         "command_triggered",
@@ -240,11 +266,25 @@ def ingest(
             merge_key=merge_key,
             interval_start=interval_start,
             interval_end=interval_end,
+            sql_backend=sql_backend,
         )
 
         if original_incremental_strategy == "delete+insert":
             dlt_source.incremental.primary_key = ()
 
+        if (
+            factory.destination_scheme in PARQUET_SUPPORTED_DESTINATIONS
+            and loader_file_format == "default"
+        ):
+            loader_file_format = "parquet"
+
+            # if the source is a JSON returning source, we cannot use Parquet loader for BigQuery
+            if factory.destination_scheme == 'bigquery' and factory.source_scheme in JSON_RETURNING_SOURCES:
+                loader_file_format = "jsonl"
+            
+        elif loader_file_format == "default":
+            loader_file_format = "jsonl"
+
         run_info = pipeline.run(
             dlt_source,
             **destination.dlt_run_params(
@@ -253,6 +293,7 @@ def ingest(
             ),
             write_disposition=incremental_strategy,  # type: ignore
             primary_key=(primary_key if primary_key and len(primary_key) > 0 else None),  # type: ignore
+            loader_file_format=loader_file_format,  # type: ignore
         )
 
         destination.post_load()

{ingestr-0.3.0 → ingestr-0.3.2}/ingestr/main_test.py

@@ -24,6 +24,8 @@ def invoke_ingest_command(
     merge_key=None,
     interval_start=None,
     interval_end=None,
+    sql_backend=None,
+    loader_file_format=None,
 ):
     args = [
         "ingest",
@@ -61,6 +63,14 @@ def invoke_ingest_command(
         args.append("--interval-end")
         args.append(interval_end)
 
+    if sql_backend:
+        args.append("--sql-backend")
+        args.append(sql_backend)
+
+    if loader_file_format:
+        args.append("--loader-file-format")
+        args.append(loader_file_format)
+
     result = runner.invoke(
         app,
         args,
@@ -93,7 +103,6 @@ def test_create_replace():
         "testschema.output",
     )
 
-    print(result.stdout)
     assert result.exit_code == 0
 
     res = conn.sql(
@@ -138,6 +147,7 @@ def test_append():
             "testschema_append.output",
             "append",
             "updated_at",
+            sql_backend="sqlalchemy",
         )
         assert res.exit_code == 0
 
@@ -194,6 +204,7 @@ def test_merge_with_primary_key():
             "merge",
             "updated_at",
             "id",
+            sql_backend="sqlalchemy",
         )
         assert res.exit_code == 0
         return res
@@ -351,6 +362,8 @@ def test_delete_insert_without_primary_key():
             "testschema_delete_insert.output",
             inc_strategy="delete+insert",
             inc_key="updated_at",
+            sql_backend="sqlalchemy",
+            loader_file_format="jsonl",
         )
         assert res.exit_code == 0
         return res
@@ -465,6 +478,8 @@ def test_delete_insert_with_timerange():
             inc_key="updated_at",
             interval_start=start_date,
             interval_end=end_date,
+            sql_backend="sqlalchemy",
+            loader_file_format="jsonl",
         )
         assert res.exit_code == 0
         return res

{ingestr-0.3.0 → ingestr-0.3.2}/ingestr/src/notion/__init__.py

@@ -9,7 +9,7 @@ from .helpers.client import NotionClient
 from .helpers.database import NotionDatabase
 
 
-@dlt.source
+@dlt.source(max_table_nesting=1)
 def notion_databases(
     database_ids: Optional[List[Dict[str, str]]] = None,
     api_key: str = dlt.secrets.value,

{ingestr-0.3.0 → ingestr-0.3.2}/ingestr/src/sources.py

@@ -41,6 +41,7 @@ class SqlSource:
             table=table_fields[-1],
             incremental=incremental,
             merge_key=kwargs.get("merge_key"),
+            backend=kwargs.get("sql_backend", "sqlalchemy"),
         )
 
         return table_instance

{ingestr-0.3.0 → ingestr-0.3.2}/ingestr/src/sources_test.py

@@ -22,10 +22,11 @@ class SqlSourceTest(unittest.TestCase):
         table = "schema.table"
 
         # monkey patch the sql_table function
-        def sql_table(credentials, schema, table, incremental, merge_key):
+        def sql_table(credentials, schema, table, incremental, merge_key, backend):
             self.assertEqual(credentials, uri)
             self.assertEqual(schema, "schema")
             self.assertEqual(table, "table")
+            self.assertEqual(backend, "sqlalchemy")
             self.assertIsNone(incremental)
             self.assertIsNone(merge_key)
             return dlt.resource()
@@ -40,10 +41,11 @@ class SqlSourceTest(unittest.TestCase):
         incremental_key = "id"
 
         # monkey patch the sql_table function
-        def sql_table(credentials, schema, table, incremental, merge_key):
+        def sql_table(credentials, schema, table, incremental, merge_key, backend):
             self.assertEqual(credentials, uri)
             self.assertEqual(schema, "schema")
             self.assertEqual(table, "table")
+            self.assertEqual(backend, "sqlalchemy")
             self.assertIsInstance(incremental, dlt.sources.incremental)
             self.assertEqual(incremental.cursor_path, incremental_key)
             self.assertIsNone(merge_key)

ingestr-0.3.2/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.3.2"

ingestr-0.3.0/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.3.0"