ingestr 0.2.6__tar.gz → 0.3.1__tar.gz

{ingestr-0.2.6 → ingestr-0.3.1}/Makefile

@@ -22,7 +22,7 @@ test-specific: venv
 
 lint-ci:
 	ruff check ingestr --fix && ruff format ingestr
-	mypy  --explicit-package-bases ingestr --config-file pyproject.toml
+	mypy --config-file pyproject.toml --explicit-package-bases ingestr
 
 lint: venv
 	. venv/bin/activate; $(MAKE) lint-ci

{ingestr-0.2.6 → ingestr-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.2.6
+Version: 0.3.1
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -16,26 +16,26 @@ Classifier: Topic :: Database
 Requires-Python: >=3.9
 Requires-Dist: cx-oracle==8.3.0
 Requires-Dist: databricks-sql-connector==2.9.3
-Requires-Dist: dlt==0.4.6
-Requires-Dist: duckdb-engine==0.11.2
-Requires-Dist: duckdb==0.10.1
+Requires-Dist: dlt==0.4.8
+Requires-Dist: duckdb-engine==0.11.5
+Requires-Dist: duckdb==0.10.2
 Requires-Dist: google-cloud-bigquery-storage==2.24.0
 Requires-Dist: pendulum==3.0.0
 Requires-Dist: psycopg2-binary==2.9.9
 Requires-Dist: py-machineid==0.5.1
-Requires-Dist: pymongo==4.6.2
+Requires-Dist: pymongo==4.6.3
 Requires-Dist: pymysql==1.1.0
 Requires-Dist: pyodbc==5.1.0
 Requires-Dist: redshift-connector==2.1.0
-Requires-Dist: rich==13.7.0
+Requires-Dist: rich==13.7.1
 Requires-Dist: rudder-sdk-python==2.1.0
-Requires-Dist: snowflake-sqlalchemy==1.5.1
-Requires-Dist: sqlalchemy-bigquery==1.10.0
+Requires-Dist: snowflake-sqlalchemy==1.5.3
+Requires-Dist: sqlalchemy-bigquery==1.11.0
 Requires-Dist: sqlalchemy-redshift==0.8.14
 Requires-Dist: sqlalchemy2-stubs==0.0.2a38
 Requires-Dist: sqlalchemy==1.4.52
 Requires-Dist: tqdm==4.66.2
-Requires-Dist: typer==0.9.0
+Requires-Dist: typer==0.12.3
 Description-Content-Type: text/markdown
 
 <div align="center">
@@ -91,20 +91,84 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
 
 ## Supported Sources & Destinations
 
-| Database             | Source | Destination |
-|----------------------|--------|-------------|
-| Postgres             | ✅      | ✅         |
-| BigQuery             | ✅      | ✅         |
-| Snowflake            | ✅      | ✅         |
-| Redshift             | ✅      | ✅         |
-| Databricks           | ✅      | ✅         |
-| DuckDB               | ✅      | ✅         |
-| Microsoft SQL Server | ✅      | ✅         |
-| Local CSV file       | ✅      | ✅         |
-| MongoDB              | ✅      | ❌         |
-| Oracle               | ✅      | ❌         |
-| SQLite               | ✅      | ❌         |
-| MySQL                | ✅      | ❌         |
+<table>
+    <tr>
+        <th></th>
+        <th>Source</th>
+        <th>Destination</th>
+    </tr>
+    <tr>
+        <td colspan="3" style='text-align:center;'><strong>Databases</strong></td>
+    </tr>
+    <tr>
+        <td>Postgres</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>BigQuery</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Snowflake</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Redshift</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Databricks</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>DuckDB</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Microsoft SQL Server</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Local CSV file</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>MongoDB</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td>Oracle</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td>SQLite</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td>MySQL</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td colspan="3" style='text-align:center;'><strong>Platforms</strong></td>
+    </tr>
+    <tr>
+        <td>Notion</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+</table>
 
 More to come soon!
 

{ingestr-0.2.6 → ingestr-0.3.1}/README.md

@@ -51,20 +51,84 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
 
 ## Supported Sources & Destinations
 
-| Database             | Source | Destination |
-|----------------------|--------|-------------|
-| Postgres             | ✅      | ✅         |
-| BigQuery             | ✅      | ✅         |
-| Snowflake            | ✅      | ✅         |
-| Redshift             | ✅      | ✅         |
-| Databricks           | ✅      | ✅         |
-| DuckDB               | ✅      | ✅         |
-| Microsoft SQL Server | ✅      | ✅         |
-| Local CSV file       | ✅      | ✅         |
-| MongoDB              | ✅      | ❌         |
-| Oracle               | ✅      | ❌         |
-| SQLite               | ✅      | ❌         |
-| MySQL                | ✅      | ❌         |
+<table>
+    <tr>
+        <th></th>
+        <th>Source</th>
+        <th>Destination</th>
+    </tr>
+    <tr>
+        <td colspan="3" style='text-align:center;'><strong>Databases</strong></td>
+    </tr>
+    <tr>
+        <td>Postgres</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>BigQuery</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Snowflake</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Redshift</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Databricks</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>DuckDB</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Microsoft SQL Server</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Local CSV file</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>MongoDB</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td>Oracle</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td>SQLite</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td>MySQL</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td colspan="3" style='text-align:center;'><strong>Platforms</strong></td>
+    </tr>
+    <tr>
+        <td>Notion</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+</table>
 
 More to come soon!
 

{ingestr-0.2.6 → ingestr-0.3.1}/docs/.vitepress/config.mjs

@@ -4,20 +4,17 @@ import { defineConfig } from "vitepress";
 export default defineConfig({
   title: "ingestr",
   description: "Ingest & copy data between any source and any destination",
-  base: '/ingestr/',
+  base: "/ingestr/",
   head: [
+    ["script", { async: "", src: "https://www.googletagmanager.com/gtag/js?id=G-MZJ20PP4MJ" }],
     [
-      'script',
-      { async: '', src: 'https://www.googletagmanager.com/gtag/js?id=G-MZJ20PP4MJ' }
-    ],
-    [
-      'script',
+      "script",
       {},
       `window.dataLayer = window.dataLayer || [];
       function gtag(){dataLayer.push(arguments);}
       gtag('js', new Date());
-      gtag('config', 'G-MZJ20PP4MJ');`
-    ]
+      gtag('config', 'G-MZJ20PP4MJ');`,
+    ],
   ],
   themeConfig: {
     // https://vitepress.dev/reference/default-theme-config
@@ -46,19 +43,31 @@ export default defineConfig({
       {
         text: "Sources & Destinations",
         items: [
-          { text: "AWS Redshift", link: "/supported-sources/redshift.md" },
-          { text: "Databricks", link: "/supported-sources/databricks.md" },
-          { text: "DuckDB", link: "/supported-sources/duckdb.md" },
-          { text: "Google BigQuery", link: "/supported-sources/bigquery.md" },
-          { text: "Local CSV Files", link: "/supported-sources/csv.md" },
-          { text: "Microsoft SQL Server", link: "/supported-sources/mssql.md" },
-          { text: "MongoDB", link: "/supported-sources/mongodb.md" },
-          { text: "MySQL", link: "/supported-sources/mysql.md" },
-          { text: "Oracle", link: "/supported-sources/oracle.md" },
           { text: "Overview", link: "/supported-sources/overview.md" },
-          { text: "Postgres", link: "/supported-sources/postgres.md" },
-          { text: "Snowflake", link: "/supported-sources/snowflake.md" },
-          { text: "SQLite", link: "/supported-sources/sqlite.md" },
+          {
+            text: "Databases",
+            collapsed: false,
+            items: [
+              { text: "AWS Redshift", link: "/supported-sources/redshift.md" },
+              { text: "Databricks", link: "/supported-sources/databricks.md" },
+              { text: "DuckDB", link: "/supported-sources/duckdb.md" },
+              { text: "Google BigQuery", link: "/supported-sources/bigquery.md" },
+              { text: "Local CSV Files", link: "/supported-sources/csv.md" },
+              { text: "Microsoft SQL Server", link: "/supported-sources/mssql.md" },
+              { text: "MongoDB", link: "/supported-sources/mongodb.md" },
+              { text: "MySQL", link: "/supported-sources/mysql.md" },
+              { text: "Oracle", link: "/supported-sources/oracle.md" },
+              { text: "Postgres", link: "/supported-sources/postgres.md" },
+              { text: "Snowflake", link: "/supported-sources/snowflake.md" },
+              { text: "SQLite", link: "/supported-sources/sqlite.md" },
+            ],
+          },
+
+          {
+            text: "Platforms",
+            collapsed: false,
+            items: [{ text: "Notion", link: "/supported-sources/notion.md" }],
+          },
         ],
       },
     ],

ingestr-0.3.1/docs/supported-sources/notion.md

@@ -0,0 +1,49 @@
+# Notion
+[Notion](https://www.notion.so/) is an all-in-one workspace for note-taking, project management, and database management.
+
+ingestr supports Notion as a source.
+
+## URI Format
+The URI format for Notion is as follows:
+
+```plaintext
+notion://?api_key=token
+```
+
+URI parameters:
+- `api_key`: the integration token used for authentication with the Notion API
+
+The URI is used to connect to the Notion API for extracting data. More details on setting up Notion integrations can be found [here](https://developers.notion.com/docs/getting-started).
+
+## Setting up a Notion Integration
+
+Notion requires a few steps to set up an integration, please follow the guide dltHub [has built here](https://dlthub.com/docs/dlt-ecosystem/verified-sources/notion#setup-guide).
+
+Once you complete the guide, you should have an API key, and the table ID to connect to. Let's say your API token is `secret_12345` and the database you'd like to connect to is `bfeaafc0c25f40a9asdasd672a9456f3`, here's a sample command that will copy the data from the Notion table into a duckdb database:
+
+```sh
+ingestr ingest --source-uri 'notion://?api_key=secret_12345' --source-table 'bfeaafc0c25f40a9asdasd672a9456f3' --dest-uri duckdb:///notion.duckdb --dest-table 'notion.output'
+```
+
+The result of this command will be a bunch of tables in the `notion.duckdb` database. The Notion integration creates a bunch of extra tables in the schema to keep track of additional information about every field in a database. You should take some time to play around with the data and understand how it's structured, and take a good look at `_dlt_parent_id` column in the tables to understand the relationships between tables.
+
+Take a look at the following Notion table:
+![an example Notion database](./images/notion_example.png)
+
+Ingesting this table using ingestr will create a bunch of new tables with quite a lot of details in them. The following query is a reconstruction of the table as it looks on Notion:
+
+```sql
+select n.text__content, s.text__content, o.properties__numerical_value__number, r.text__content
+from notion.output o
+         join notion.output__properties__name__title n on n._dlt_parent_id = o._dlt_id
+         join notion.output__properties__another_col__rich_text r on r._dlt_parent_id = o._dlt_id
+         join notion.output__properties__second_value__rich_text s on s._dlt_parent_id = o._dlt_id
+order by 1;
+```
+
+Take this as a starting point and play around with the data.
+
+
+> [!CAUTION]
+> Notion does not support incremental loading, which means every time you run the command, it will copy the entire table from Notion to the destination. This can be slow for large tables.
+

ingestr-0.3.1/docs/supported-sources/overview.md

@@ -0,0 +1,84 @@
+# Supported Sources & Destinations
+ingestr supports the following sources and destinations:
+
+
+<table>
+    <tr>
+        <th></th>
+        <th>Source</th>
+        <th>Destination</th>
+    </tr>
+    <tr>
+        <td colspan="3" style='text-align:center;'><strong>Databases</strong></td>
+    </tr>
+    <tr>
+        <td>Postgres</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>BigQuery</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Snowflake</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Redshift</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Databricks</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>DuckDB</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Microsoft SQL Server</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Local CSV file</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>MongoDB</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td>Oracle</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td>SQLite</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td>MySQL</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td colspan="3" style='text-align:center;'><strong>Platforms</strong></td>
+    </tr>
+    <tr>
+        <td>Notion</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+</table>
+
+More to come soon!

{ingestr-0.2.6 → ingestr-0.3.1}/docs/supported-sources/snowflake.md

@@ -7,7 +7,7 @@ ingestr supports Snowflake as both a source and destination.
 The URI format for Snowflake is as follows:
 
 ```plaintext
-snowflake://user:password@account/dbname?warehouse=COMPUTE_WH
+snowflake://user:password@account/dbname?warehouse=COMPUTE_WH&role=data_scientist
 ```
 
 URI parameters:
@@ -15,6 +15,7 @@ URI parameters:
 - `password`: the password for the user
 - `account`: your Snowflake account identifier
 - `dbname`: the name of the database to connect to
-- `warehouse`: the name of the warehouse to use
+- `warehouse`: the name of the warehouse to use (optional)
+- `role`: the name of the role to use (optional)
 
 The same URI structure can be used both for sources and destinations. You can read more about SQLAlchemy's Snowflake dialect [here](https://docs.snowflake.com/en/developer-guide/python-connector/sqlalchemy#connection-parameters).

{ingestr-0.2.6 → ingestr-0.3.1}/ingestr/main.py

@@ -5,7 +5,7 @@ from typing import Optional
 import dlt
 import humanize
 import typer
-from dlt.common.runtime.collector import Collector
+from dlt.common.runtime.collector import Collector, LogCollector
 from rich.console import Console
 from rich.status import Status
 from typing_extensions import Annotated
@@ -32,10 +32,17 @@ DATE_FORMATS = [
     "%Y-%m-%dT%H:%M:%S.%f%z",
 ]
 
+# https://dlthub.com/docs/dlt-ecosystem/file-formats/parquet#supported-destinations
+PARQUET_SUPPORTED_DESTINATIONS = [
+    "bigquery",
+    "duckdb",
+    "snowflake",
+    "databricks",
+    "synapse",
+]
 
-class SpinnerCollector(Collector):
-    """A Collector that shows progress with `tqdm` progress bars"""
 
+class SpinnerCollector(Collector):
     status: Status
     current_step: str
     started: bool
@@ -150,6 +157,27 @@ def ingest(
             envvar="FULL_REFRESH",
         ),
     ] = False,  # type: ignore
+    progress: Annotated[
+        Optional[str],
+        typer.Option(
+            help="The progress display type, must be one of 'interactive', 'log'",
+            envvar="PROGRESS",
+        ),
+    ] = "interactive",  # type: ignore
+    sql_backend: Annotated[
+        Optional[str],
+        typer.Option(
+            help="The SQL backend to use, must be one of 'sqlalchemy', 'pyarrow'",
+            envvar="SQL_BACKEND",
+        ),
+    ] = "pyarrow",  # type: ignore
+    loader_file_format: Annotated[
+        Optional[str],
+        typer.Option(
+            help="The file format to use when loading data, must be one of 'jsonl', 'parquet', 'default'",
+            envvar="LOADER_FILE_FORMAT",
+        ),
+    ] = "default",  # type: ignore
 ):
     track(
         "command_triggered",
@@ -186,12 +214,16 @@ def ingest(
         m = hashlib.sha256()
         m.update(dest_table.encode("utf-8"))
 
+        progressInstance: Collector = SpinnerCollector()
+        if progress == "log":
+            progressInstance = LogCollector()
+
         pipeline = dlt.pipeline(
             pipeline_name=m.hexdigest(),
             destination=destination.dlt_dest(
                 uri=dest_uri,
             ),
-            progress=SpinnerCollector(),
+            progress=progressInstance,
             pipelines_dir="pipeline_data",
             full_refresh=full_refresh,
         )
@@ -231,11 +263,20 @@ def ingest(
             merge_key=merge_key,
             interval_start=interval_start,
             interval_end=interval_end,
+            sql_backend=sql_backend,
         )
 
         if original_incremental_strategy == "delete+insert":
             dlt_source.incremental.primary_key = ()
 
+        if (
+            factory.destination_scheme in PARQUET_SUPPORTED_DESTINATIONS
+            and loader_file_format == "default"
+        ):
+            loader_file_format = "parquet"
+        elif loader_file_format == "default":
+            loader_file_format = "jsonl"
+
         run_info = pipeline.run(
             dlt_source,
             **destination.dlt_run_params(
@@ -244,6 +285,7 @@ def ingest(
             ),
             write_disposition=incremental_strategy,  # type: ignore
             primary_key=(primary_key if primary_key and len(primary_key) > 0 else None),  # type: ignore
+            loader_file_format=loader_file_format,  # type: ignore
         )
 
         destination.post_load()