ingestr 0.13.19__tar.gz → 0.13.21__tar.gz

{ingestr-0.13.19 → ingestr-0.13.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.19
+Version: 0.13.21
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -81,6 +81,8 @@ Requires-Dist: hdbcli==2.23.27
 Requires-Dist: hexbytes==1.3.0
 Requires-Dist: httplib2==0.22.0
 Requires-Dist: humanize==4.12.1
+Requires-Dist: ibm-db-sa==0.4.1
+Requires-Dist: ibm-db==3.2.6
 Requires-Dist: idna==3.10
 Requires-Dist: inflection==0.5.1
 Requires-Dist: isodate==0.7.2

{ingestr-0.13.19 → ingestr-0.13.21}/docs/.vitepress/config.mjs

@@ -40,7 +40,6 @@ export default defineConfig({
             text: "Incremental Loading",
             link: "/getting-started/incremental-loading.md",
           },
-          { text: "Load Kinesis Data to BigQuery", link: "/tutorials/load-kinesis-bigquery.md" },
           { text: "Telemetry", link: "/getting-started/telemetry.md" },
         ],
       },
@@ -51,6 +50,14 @@ export default defineConfig({
           { text: "example-uris", link: "/commands/example-uris.md" },
         ],
       },
+      {
+        text: "Tutorials",
+        items: [
+          { text: "Load Kinesis Data to BigQuery", link: "/tutorials/load-kinesis-bigquery.md" },
+          { text: "Load Personio Data to DuckDB", link: "/tutorials/load-personio-duckdb.md" },
+          { text: "Load Stripe Data to Postgres", link: "/tutorials/load-stripe-postgres.md" },
+        ],
+      },
       {
         text: "Sources & Destinations",
         items: [
@@ -68,6 +75,7 @@ export default defineConfig({
                 text: "Google BigQuery",
                 link: "/supported-sources/bigquery.md",
               },
+              { text: "IBM Db2", link: "/supported-sources/db2.md" },
               { text: "Kafka", link: "/supported-sources/kafka.md" },
               { text: "Local CSV Files", link: "/supported-sources/csv.md" },
               {

ingestr-0.13.21/docs/supported-sources/db2.md

@@ -0,0 +1,20 @@
+# IBM Db2
+[IBM Db2](https://www.ibm.com/db2) is a high-performance, enterprise-grade relational database system designed for reliability, scalability, and transactional integrity.
+
+ingestr supports IBM Db2 as a source.
+
+## URI format
+The URI format for DB2 is as follows:
+
+```plaintext
+db2://<username>:<password>@<host>:<port>/<database-name>
+```
+
+URI parameters:
+- `username`: The username to connect to the database
+- `password`: The password for the user
+- `host`: The host address of the database server
+- `port`: The port number the database server is listening
+- `database-name`: the name of the database to connect to
+
+The same URI structure can be used both for sources and destinations. You can read more about SQLAlchemy's Db2 dialect [here](https://github.com/ibmdb/python-ibmdbsa?tab=readme-ov-file#connecting).

ingestr-0.13.21/docs/tutorials/load-personio-duckdb.md

@@ -0,0 +1,95 @@
+# Load Data from Personio to DuckDB
+
+Welcome! 👋  
+ This beginner-friendly guide will help you load data from `Personio` into `DuckDB` using ingestr — a simple yet powerful command-line tool. No prior experience is needed, and best of all, no coding required!
+
+By the end of this guide, you'll have your Personio data securely stored in DuckDB. But before we dive in, let’s take a quick look at `ingestr`
+
+## Overview of ingestr
+`ingestr` is a command-line tool that simplifies data ingestion by allowing users to load data from a source to a destination using simple command-line flags.
+
+### `ingestr` Command
+```bash
+ingestr ingest \
+   --source-uri '<your-source-uri-here>' \
+   --source-table '<your-schema>.<your-table>' \
+   --dest-uri '<your-destination-uri-here>' \
+   --dest-table '<your-schema>.<your-table>'
+```
+- `ingestr ingest`: Executes the data ingestion process.
+- `--source-uri TEXT`: Specifies the URI of the data source.
+- `--dest-uri TEXT`: Specifies the URI of the destination.
+- `--source-table TEXT`: Defines the table to fetch data from.
+- `--dest-table TEXT`: Specifies the destination table. If not provided, it defaults to `--source-table`.
+
+With this command, we connect to the source, retrieve the specified data, and load it into the destination database.
+## Let's Load Data from Personio to DuckDB Together!
+
+Personio is a human resources management platform that helps businesses handle recruitment and employee data. To analyze this data, you may need to load it into an analytics database like DuckDB. `ingestr` makes this process simple.
+
+### Step 1: Install `ingestr`
+Ensure `ingestr` is installed. If not, follow the installation guide [here](../getting-started/quickstart.md#Installation).
+
+### Step 2: Get Personio Credentials
+Personio will be our data source.
+1. Log in to your Personio account.
+2. Ensure your user has API access.
+3. Navigate to **Settings > Integrations > API Credentials**.
+4. Click **Generate new credentials**.
+5. Assign **read access** to required attributes (e.g., last name, last modified).
+6. Copy the generated `client ID` and `client secret`.
+
+For more details, refer to [Personio API documentation](https://developer.personio.de/docs/getting-started-with-the-personio-api#21-employee-attendance-and-absence-endpoints).
+
+### Step 3: Install DuckDB
+DuckDB will be our data destination. If it’s not already installed, you can install it using pip:
+```bash
+pip install duckdb
+```  
+Alternatively, you can download the latest version from the [official DuckDB website](https://duckdb.org/docs/installation/?version=stable&environment=cli&platform=macos&download_method=direct).
+
+### Step 4: Run the `ingestr` Command
+Execute the following command to load data from Personio to DuckDB:
+```bash
+ingestr ingest \
+    --source-uri 'personio://?client_id=<YOUR_CLIENT_ID>&client_secret=<YOUR_CLIENT_SECRET>' \
+    --source-table 'employees' \
+    --dest-uri 'duckdb:///personio.duckdb' \
+    --dest-table 'dest.employees'
+```
+- `--source-uri 'personio://?client_id=<ID>&client_secret=<SECRET>'`: Connects to Personio using API credentials.
+- `--source-table 'employees'`: Specifies the table to fetch data from Personio.
+- `--dest-uri 'duckdb:///personio.duckdb'`: Specifies DuckDB as the destination database.
+- `--dest-table 'dest.employees'`: Defines where the data will be stored in DuckDB.
+
+
+## Verify Data in DuckDB
+Once the command runs successfully, your Personio data will be available in DuckDB. Follow these steps to verify the data:
+
+### Step 1: Open DuckDB
+If DuckDB is installed, you can open it using the following command:
+```bash
+duckdb personio.duckdb
+```
+
+### Step 2: List Available Tables
+To check if the `employees` table has been created, run:
+```sql
+.tables;
+```
+
+### Step 3: View Data in the `employees` Table
+To display all data from the `employees` table, run:
+```sql
+SELECT * FROM dest.employees;
+```
+
+### Step 4: Validate Data
+Ensure that the retrieved data matches what was expected from Personio.
+
+Example output:
+
+![Personio to DuckDB](../media/personio_duckdb.png)
+
+🎉 **Congratulations!** You have successfully loaded data from Personio to DuckDB using `ingestr`. 
+

ingestr-0.13.21/docs/tutorials/load-stripe-postgres.md

@@ -0,0 +1,101 @@
+# Load Data from Stripe to PostgreSQL
+
+Welcome! 👋 This beginner-friendly guide will help you load data from `Stripe` into `PostgreSQL` using ingestr — a simple yet powerful command-line tool. No prior experience is needed, and best of all, no coding required! 🚀
+
+By the end of this guide, you'll have your Stripe data securely stored in PostgreSQL. But before we dive in, let’s take a quick look at ingestr.
+
+## Overview of `ingestr`
+`ingestr` is a command-line tool that simplifies data ingestion by allowing users to load data from a source to a destination.
+
+### `ingestr` Command
+```bash
+ingestr ingest \
+   --source-uri '<your-source-uri-here>' \
+   --source-table '<your-schema>.<your-table>' \
+   --dest-uri '<your-destination-uri-here>' \
+   --dest-table '<your-schema>.<your-table>'
+```
+- `ingestr ingest`: Executes the data ingestion process.
+- `--source-uri TEXT`: Defines the source database URI.
+- `--dest-uri TEXT`: Defines the destination database URI.
+- `--source-table TEXT`: Specifies the table to fetch data from.
+- `--dest-table TEXT`: Defines where the data should be stored (defaults to `--source-table` if not specified).
+
+With this command, we connect to the source, retrieve the specified data, and load it into the destination database.
+
+## Let's Load Data from Stripe into PostgreSQL Together!
+
+Stripe is a popular payment processing platform used by e-commerce and SaaS applications. If you need to analyze Stripe data, you can load it into PostgreSQL, a reliable and widely used relational database.
+
+## Step 1: Install `ingestr`
+Ensure `ingestr` is installed on your system. If not, follow the installation guide [here](../getting-started/quickstart.md#Installation).
+
+```bash
+pip install ingestr
+```
+
+## Step 2: Retrieve Stripe API Credentials
+Stripe will be our **data source**. To retrieve your API credentials:
+
+1. Log in to your **Stripe Dashboard**: [dashboard.stripe.com](https://dashboard.stripe.com)
+2. Navigate to **Developers** → **API Keys**.
+3. Under **Standard Keys**, find:
+   - **Secret Key**: Required for server-to-server requests
+4. Click **Reveal Key** next to your **Secret Key** and copy it.
+
+For detailed API documentation, visit the [Stripe API Docs](https://stripe.com/docs/api).
+
+## Step 3: Retrieve PostgreSQL Credentials
+In this example, we are using a **cloud-hosted PostgreSQL instance**.
+1. Log in to your cloud provider (e.g., [Neon](https://neon.tech/)).
+2. Navigate to your **PostgreSQL instance** and retrieve:
+   - **Host**: Example - `your-db-instance.aws.com`
+   - **Port**: Default is `5432`
+   - **Database Name**: The database where Stripe data will be stored.
+   - **Username & Password**: Credentials used to connect to PostgreSQL.
+   - **sslmode**: Required depending on the security settings of your PostgreSQL instance
+
+## Step 4: Run the `ingestr` Command
+Execute the following command to load data from **Stripe to PostgreSQL**:
+
+```bash
+ingestr ingest \
+    --source-uri 'stripe://?api_key=<YOUR_STRIPE_SECRET_KEY>' \
+    --source-table 'event' \
+    --dest-uri 'postgresql://<username>:<password>@<host>:<port>/<database-name>?sslmode=<sslmode>' \
+    --dest-table 'dest.events'
+```
+- `--source-uri 'stripe://?api_key=<YOUR_STRIPE_SECRET_KEY>'`: Connects to Stripe using secret key .
+- `--source-table 'event'`: Specifies the data to fetch data from Stripe.
+- `--dest-uri 'postgresql://<username>:<password>@<host>:<port>/<database-name>?sslmode=<sslmode>'`: Connects to PostgreSQL as the destination database using the provided credentials.
+- `--dest-table 'dest.events'`: Defines where the data will be stored in Postgres.
+
+## Step 5: Verify Data in PostgreSQL via DBeaver
+
+Once the command runs successfully, let's verify that the Stripe data has been loaded into PostgreSQL.
+
+### 1: Connect to PostgreSQL via DBeaver
+1. **Open DBeaver** and go to **Database** → **New Connection**.
+2. Select **PostgreSQL**.
+3. Enter your **PostgreSQL credentials**:
+   - **Host**: `<your-host>`
+   - **Port**: `5432` (default)
+   - **Database**: `<your-database>`
+   - **Username**: `<your-username>`
+   - **Password**: `<your-password>`
+4. Click **Test Connection** to verify.
+5. If successful, click **Finish**.
+
+### 2: Browse Tables
+1. Expand your **PostgreSQL connection** in the **Database Navigator**.
+2. Expand **Schemas** → **Public**.
+3. Expand **Tables** and find `events`.
+
+### 3: View Data in DBeaver
+1. Right-click the `events` table.
+2. Click **View Data** → **All Rows**.
+3. Ensure that the retrieved data matches what was expected from Stripe.
+
+![Stripe to Postgresql](../media/stripe_postgres.png)
+🎉 Congratulations!
+Your Stripe data is now securely stored in PostgreSQL 🚀

{ingestr-0.13.19 → ingestr-0.13.21}/ingestr/main.py

@@ -127,40 +127,44 @@ class SqlReflectionLevel(str, Enum):
 def ingest(
     source_uri: Annotated[
         str,
-        typer.Option(help="The URI of the [green]source[/green]", envvar="SOURCE_URI"),
+        typer.Option(
+            help="The URI of the [green]source[/green]",
+            envvar=["SOURCE_URI", "INGESTR_SOURCE_URI"],
+        ),
     ],  # type: ignore
     dest_uri: Annotated[
         str,
         typer.Option(
-            help="The URI of the [cyan]destination[/cyan]", envvar="DESTINATION_URI"
+            help="The URI of the [cyan]destination[/cyan]",
+            envvar=["DESTINATION_URI", "INGESTR_DESTINATION_URI"],
         ),
     ],  # type: ignore
     source_table: Annotated[
         str,
         typer.Option(
             help="The table name in the [green]source[/green] to fetch",
-            envvar="SOURCE_TABLE",
+            envvar=["SOURCE_TABLE", "INGESTR_SOURCE_TABLE"],
         ),
     ],  # type: ignore
     dest_table: Annotated[
         str,
         typer.Option(
             help="The table in the [cyan]destination[/cyan] to save the data into",
-            envvar="DESTINATION_TABLE",
+            envvar=["DESTINATION_TABLE", "INGESTR_DESTINATION_TABLE"],
         ),
     ] = None,  # type: ignore
     incremental_key: Annotated[
         Optional[str],
         typer.Option(
             help="The incremental key from the table to be used for incremental strategies",
-            envvar="INCREMENTAL_KEY",
+            envvar=["INCREMENTAL_KEY", "INGESTR_INCREMENTAL_KEY"],
         ),
     ] = None,  # type: ignore
     incremental_strategy: Annotated[
         IncrementalStrategy,
         typer.Option(
             help="The incremental strategy to use",
-            envvar="INCREMENTAL_STRATEGY",
+            envvar=["INCREMENTAL_STRATEGY", "INGESTR_INCREMENTAL_STRATEGY"],
         ),
     ] = IncrementalStrategy.create_replace,  # type: ignore
     interval_start: Annotated[
@@ -168,7 +172,7 @@ def ingest(
         typer.Option(
             help="The start of the interval the incremental key will cover",
             formats=DATE_FORMATS,
-            envvar="INTERVAL_START",
+            envvar=["INTERVAL_START", "INGESTR_INTERVAL_START"],
         ),
     ] = None,  # type: ignore
     interval_end: Annotated[
@@ -176,126 +180,126 @@ def ingest(
         typer.Option(
             help="The end of the interval the incremental key will cover",
             formats=DATE_FORMATS,
-            envvar="INTERVAL_END",
+            envvar=["INTERVAL_END", "INGESTR_INTERVAL_END"],
         ),
     ] = None,  # type: ignore
     primary_key: Annotated[
         Optional[list[str]],
         typer.Option(
             help="The key that will be used to deduplicate the resulting table",
-            envvar="PRIMARY_KEY",
+            envvar=["PRIMARY_KEY", "INGESTR_PRIMARY_KEY"],
         ),
     ] = None,  # type: ignore
     partition_by: Annotated[
         Optional[str],
         typer.Option(
             help="The partition key to be used for partitioning the destination table",
-            envvar="PARTITION_BY",
+            envvar=["PARTITION_BY", "INGESTR_PARTITION_BY"],
         ),
     ] = None,  # type: ignore
     cluster_by: Annotated[
         Optional[str],
         typer.Option(
             help="The clustering key to be used for clustering the destination table, not every destination supports clustering.",
-            envvar="CLUSTER_BY",
+            envvar=["CLUSTER_BY", "INGESTR_CLUSTER_BY"],
         ),
     ] = None,  # type: ignore
     yes: Annotated[
         Optional[bool],
         typer.Option(
             help="Skip the confirmation prompt and ingest right away",
-            envvar="SKIP_CONFIRMATION",
+            envvar=["SKIP_CONFIRMATION", "INGESTR_SKIP_CONFIRMATION"],
         ),
     ] = False,  # type: ignore
     full_refresh: Annotated[
         bool,
         typer.Option(
             help="Ignore the state and refresh the destination table completely",
-            envvar="FULL_REFRESH",
+            envvar=["FULL_REFRESH", "INGESTR_FULL_REFRESH"],
         ),
     ] = False,  # type: ignore
     progress: Annotated[
         Progress,
         typer.Option(
             help="The progress display type, must be one of 'interactive', 'log'",
-            envvar="PROGRESS",
+            envvar=["PROGRESS", "INGESTR_PROGRESS"],
         ),
     ] = Progress.interactive,  # type: ignore
     sql_backend: Annotated[
         SqlBackend,
         typer.Option(
             help="The SQL backend to use",
-            envvar="SQL_BACKEND",
+            envvar=["SQL_BACKEND", "INGESTR_SQL_BACKEND"],
         ),
     ] = SqlBackend.pyarrow,  # type: ignore
     loader_file_format: Annotated[
         Optional[LoaderFileFormat],
         typer.Option(
             help="The file format to use when loading data",
-            envvar="LOADER_FILE_FORMAT",
+            envvar=["LOADER_FILE_FORMAT", "INGESTR_LOADER_FILE_FORMAT"],
         ),
     ] = None,  # type: ignore
     page_size: Annotated[
         Optional[int],
         typer.Option(
             help="The page size to be used when fetching data from SQL sources",
-            envvar="PAGE_SIZE",
+            envvar=["PAGE_SIZE", "INGESTR_PAGE_SIZE"],
         ),
     ] = 50000,  # type: ignore
     loader_file_size: Annotated[
         Optional[int],
         typer.Option(
             help="The file size to be used by the loader to split the data into multiple files. This can be set independent of the page size, since page size is used for fetching the data from the sources whereas this is used for the processing/loading part.",
-            envvar="LOADER_FILE_SIZE",
+            envvar=["LOADER_FILE_SIZE", "INGESTR_LOADER_FILE_SIZE"],
         ),
     ] = 100000,  # type: ignore
     schema_naming: Annotated[
         SchemaNaming,
         typer.Option(
             help="The naming convention to use when moving the tables from source to destination. The default behavior is explained here: https://dlthub.com/docs/general-usage/schema#naming-convention",
-            envvar="SCHEMA_NAMING",
+            envvar=["SCHEMA_NAMING", "INGESTR_SCHEMA_NAMING"],
         ),
     ] = SchemaNaming.default,  # type: ignore
     pipelines_dir: Annotated[
         Optional[str],
         typer.Option(
             help="The path to store dlt-related pipeline metadata. By default, ingestr will create a temporary directory and delete it after the execution is done in order to make retries stateless.",
-            envvar="PIPELINES_DIR",
+            envvar=["PIPELINES_DIR", "INGESTR_PIPELINES_DIR"],
         ),
     ] = None,  # type: ignore
     extract_parallelism: Annotated[
         Optional[int],
         typer.Option(
             help="The number of parallel jobs to run for extracting data from the source, only applicable for certain sources",
-            envvar="EXTRACT_PARALLELISM",
+            envvar=["EXTRACT_PARALLELISM", "INGESTR_EXTRACT_PARALLELISM"],
         ),
     ] = 5,  # type: ignore
     sql_reflection_level: Annotated[
         SqlReflectionLevel,
         typer.Option(
             help="The reflection level to use when reflecting the table schema from the source",
-            envvar="SQL_REFLECTION_LEVEL",
+            envvar=["SQL_REFLECTION_LEVEL", "INGESTR_SQL_REFLECTION_LEVEL"],
         ),
     ] = SqlReflectionLevel.full,  # type: ignore
     sql_limit: Annotated[
         Optional[int],
         typer.Option(
             help="The limit to use when fetching data from the source",
-            envvar="SQL_LIMIT",
+            envvar=["SQL_LIMIT", "INGESTR_SQL_LIMIT"],
         ),
     ] = None,  # type: ignore
     sql_exclude_columns: Annotated[
         Optional[list[str]],
         typer.Option(
             help="The columns to exclude from the source table",
-            envvar="SQL_EXCLUDE_COLUMNS",
+            envvar=["SQL_EXCLUDE_COLUMNS", "INGESTR_SQL_EXCLUDE_COLUMNS"],
         ),
     ] = [],  # type: ignore
     columns: Annotated[
         Optional[list[str]],
         typer.Option(
             help="The column types to be used for the destination table in the format of 'column_name:column_type'",
-            envvar="COLUMNS",
+            envvar=["COLUMNS", "INGESTR_COLUMNS"],
         ),
     ] = None,  # type: ignore
 ):

ingestr-0.13.21/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.21"

{ingestr-0.13.19 → ingestr-0.13.21}/ingestr/src/factory.py

@@ -72,6 +72,7 @@ SQL_SOURCE_SCHEMES = [
     "hana",
     "clickhouse",
     "databricks",
+    "db2",
 ]
 
 

{ingestr-0.13.19 → ingestr-0.13.21}/ingestr/src/sources.py

@@ -177,6 +177,9 @@ class SqlSource:
                 scheme="clickhouse+native",
                 query=urlencode(query_params, doseq=True),
             ).geturl()
+         
+        if uri.startswith("db2://"):
+            uri = uri.replace("db2://", "db2+ibm_db://")
 
         query_adapters = []
         if kwargs.get("sql_limit"):

{ingestr-0.13.19 → ingestr-0.13.21}/requirements.in

@@ -47,3 +47,5 @@ clickhouse-connect==0.8.14
 clickhouse-driver==0.2.9
 clickhouse-sqlalchemy==0.2.7
 zstd==1.5.6.5
+ibm-db==3.2.6
+ibm-db-sa==0.4.1

{ingestr-0.13.19 → ingestr-0.13.21}/requirements.txt

@@ -207,6 +207,12 @@ httplib2==0.22.0
     #   google-auth-httplib2
 humanize==4.12.1
     # via dlt
+ibm-db==3.2.6
+    # via
+    #   -r requirements.in
+    #   ibm-db-sa
+ibm-db-sa==0.4.1
+    # via -r requirements.in
 idna==3.10
     # via
     #   requests
@@ -476,6 +482,7 @@ sqlalchemy==1.4.52
     #   databricks-sql-connector
     #   databricks-sqlalchemy
     #   duckdb-engine
+    #   ibm-db-sa
     #   snowflake-sqlalchemy
     #   sqlalchemy-bigquery
     #   sqlalchemy-hana

ingestr-0.13.19/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.19"