ingestr 0.13.19__tar.gz → 0.13.20__tar.gz

{ingestr-0.13.19 → ingestr-0.13.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.19
+Version: 0.13.20
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -81,6 +81,8 @@ Requires-Dist: hdbcli==2.23.27
 Requires-Dist: hexbytes==1.3.0
 Requires-Dist: httplib2==0.22.0
 Requires-Dist: humanize==4.12.1
+Requires-Dist: ibm-db-sa==0.4.1
+Requires-Dist: ibm-db==3.2.6
 Requires-Dist: idna==3.10
 Requires-Dist: inflection==0.5.1
 Requires-Dist: isodate==0.7.2

{ingestr-0.13.19 → ingestr-0.13.20}/docs/.vitepress/config.mjs

@@ -40,7 +40,6 @@ export default defineConfig({
             text: "Incremental Loading",
             link: "/getting-started/incremental-loading.md",
           },
-          { text: "Load Kinesis Data to BigQuery", link: "/tutorials/load-kinesis-bigquery.md" },
           { text: "Telemetry", link: "/getting-started/telemetry.md" },
         ],
       },
@@ -51,6 +50,14 @@ export default defineConfig({
           { text: "example-uris", link: "/commands/example-uris.md" },
         ],
       },
+      {
+        text: "Tutorials",
+        items: [
+          { text: "Load Kinesis Data to BigQuery", link: "/tutorials/load-kinesis-bigquery.md" },
+          { text: "Load Personio Data to DuckDB", link: "/tutorials/load-personio-duckdb.md" },
+          { text: "Load Stripe Data to Postgres", link: "/tutorials/load-stripe-postgres.md" },
+        ],
+      },
       {
         text: "Sources & Destinations",
         items: [
@@ -68,6 +75,7 @@ export default defineConfig({
                 text: "Google BigQuery",
                 link: "/supported-sources/bigquery.md",
               },
+              { text: "IBM Db2", link: "/supported-sources/db2.md" },
               { text: "Kafka", link: "/supported-sources/kafka.md" },
               { text: "Local CSV Files", link: "/supported-sources/csv.md" },
               {

ingestr-0.13.20/docs/supported-sources/db2.md

@@ -0,0 +1,20 @@
+# IBM Db2
+[IBM Db2](https://www.ibm.com/db2) is a high-performance, enterprise-grade relational database system designed for reliability, scalability, and transactional integrity.
+
+ingestr supports IBM Db2 as a source.
+
+## URI format
+The URI format for DB2 is as follows:
+
+```plaintext
+db2://<username>:<password>@<host>:<port>/<database-name>
+```
+
+URI parameters:
+- `username`: The username to connect to the database
+- `password`: The password for the user
+- `host`: The host address of the database server
+- `port`: The port number the database server is listening
+- `database-name`: the name of the database to connect to
+
+The same URI structure can be used both for sources and destinations. You can read more about SQLAlchemy's Db2 dialect [here](https://github.com/ibmdb/python-ibmdbsa?tab=readme-ov-file#connecting).

ingestr-0.13.20/docs/tutorials/load-personio-duckdb.md

@@ -0,0 +1,95 @@
+# Load Data from Personio to DuckDB
+
+Welcome! 👋  
+ This beginner-friendly guide will help you load data from `Personio` into `DuckDB` using ingestr — a simple yet powerful command-line tool. No prior experience is needed, and best of all, no coding required!
+
+By the end of this guide, you'll have your Personio data securely stored in DuckDB. But before we dive in, let’s take a quick look at `ingestr`
+
+## Overview of ingestr
+`ingestr` is a command-line tool that simplifies data ingestion by allowing users to load data from a source to a destination using simple command-line flags.
+
+### `ingestr` Command
+```bash
+ingestr ingest \
+   --source-uri '<your-source-uri-here>' \
+   --source-table '<your-schema>.<your-table>' \
+   --dest-uri '<your-destination-uri-here>' \
+   --dest-table '<your-schema>.<your-table>'
+```
+- `ingestr ingest`: Executes the data ingestion process.
+- `--source-uri TEXT`: Specifies the URI of the data source.
+- `--dest-uri TEXT`: Specifies the URI of the destination.
+- `--source-table TEXT`: Defines the table to fetch data from.
+- `--dest-table TEXT`: Specifies the destination table. If not provided, it defaults to `--source-table`.
+
+With this command, we connect to the source, retrieve the specified data, and load it into the destination database.
+## Let's Load Data from Personio to DuckDB Together!
+
+Personio is a human resources management platform that helps businesses handle recruitment and employee data. To analyze this data, you may need to load it into an analytics database like DuckDB. `ingestr` makes this process simple.
+
+### Step 1: Install `ingestr`
+Ensure `ingestr` is installed. If not, follow the installation guide [here](../getting-started/quickstart.md#Installation).
+
+### Step 2: Get Personio Credentials
+Personio will be our data source.
+1. Log in to your Personio account.
+2. Ensure your user has API access.
+3. Navigate to **Settings > Integrations > API Credentials**.
+4. Click **Generate new credentials**.
+5. Assign **read access** to required attributes (e.g., last name, last modified).
+6. Copy the generated `client ID` and `client secret`.
+
+For more details, refer to [Personio API documentation](https://developer.personio.de/docs/getting-started-with-the-personio-api#21-employee-attendance-and-absence-endpoints).
+
+### Step 3: Install DuckDB
+DuckDB will be our data destination. If it’s not already installed, you can install it using pip:
+```bash
+pip install duckdb
+```  
+Alternatively, you can download the latest version from the [official DuckDB website](https://duckdb.org/docs/installation/?version=stable&environment=cli&platform=macos&download_method=direct).
+
+### Step 4: Run the `ingestr` Command
+Execute the following command to load data from Personio to DuckDB:
+```bash
+ingestr ingest \
+    --source-uri 'personio://?client_id=<YOUR_CLIENT_ID>&client_secret=<YOUR_CLIENT_SECRET>' \
+    --source-table 'employees' \
+    --dest-uri 'duckdb:///personio.duckdb' \
+    --dest-table 'dest.employees'
+```
+- `--source-uri 'personio://?client_id=<ID>&client_secret=<SECRET>'`: Connects to Personio using API credentials.
+- `--source-table 'employees'`: Specifies the table to fetch data from Personio.
+- `--dest-uri 'duckdb:///personio.duckdb'`: Specifies DuckDB as the destination database.
+- `--dest-table 'dest.employees'`: Defines where the data will be stored in DuckDB.
+
+
+## Verify Data in DuckDB
+Once the command runs successfully, your Personio data will be available in DuckDB. Follow these steps to verify the data:
+
+### Step 1: Open DuckDB
+If DuckDB is installed, you can open it using the following command:
+```bash
+duckdb personio.duckdb
+```
+
+### Step 2: List Available Tables
+To check if the `employees` table has been created, run:
+```sql
+.tables;
+```
+
+### Step 3: View Data in the `employees` Table
+To display all data from the `employees` table, run:
+```sql
+SELECT * FROM dest.employees;
+```
+
+### Step 4: Validate Data
+Ensure that the retrieved data matches what was expected from Personio.
+
+Example output:
+
+![Personio to DuckDB](../media/personio_duckdb.png)
+
+🎉 **Congratulations!** You have successfully loaded data from Personio to DuckDB using `ingestr`. 
+

ingestr-0.13.20/docs/tutorials/load-stripe-postgres.md

@@ -0,0 +1,101 @@
+# Load Data from Stripe to PostgreSQL
+
+Welcome! 👋 This beginner-friendly guide will help you load data from `Stripe` into `PostgreSQL` using ingestr — a simple yet powerful command-line tool. No prior experience is needed, and best of all, no coding required! 🚀
+
+By the end of this guide, you'll have your Stripe data securely stored in PostgreSQL. But before we dive in, let’s take a quick look at ingestr.
+
+## Overview of `ingestr`
+`ingestr` is a command-line tool that simplifies data ingestion by allowing users to load data from a source to a destination.
+
+### `ingestr` Command
+```bash
+ingestr ingest \
+   --source-uri '<your-source-uri-here>' \
+   --source-table '<your-schema>.<your-table>' \
+   --dest-uri '<your-destination-uri-here>' \
+   --dest-table '<your-schema>.<your-table>'
+```
+- `ingestr ingest`: Executes the data ingestion process.
+- `--source-uri TEXT`: Defines the source database URI.
+- `--dest-uri TEXT`: Defines the destination database URI.
+- `--source-table TEXT`: Specifies the table to fetch data from.
+- `--dest-table TEXT`: Defines where the data should be stored (defaults to `--source-table` if not specified).
+
+With this command, we connect to the source, retrieve the specified data, and load it into the destination database.
+
+## Let's Load Data from Stripe into PostgreSQL Together!
+
+Stripe is a popular payment processing platform used by e-commerce and SaaS applications. If you need to analyze Stripe data, you can load it into PostgreSQL, a reliable and widely used relational database.
+
+## Step 1: Install `ingestr`
+Ensure `ingestr` is installed on your system. If not, follow the installation guide [here](../getting-started/quickstart.md#Installation).
+
+```bash
+pip install ingestr
+```
+
+## Step 2: Retrieve Stripe API Credentials
+Stripe will be our **data source**. To retrieve your API credentials:
+
+1. Log in to your **Stripe Dashboard**: [dashboard.stripe.com](https://dashboard.stripe.com)
+2. Navigate to **Developers** → **API Keys**.
+3. Under **Standard Keys**, find:
+   - **Secret Key**: Required for server-to-server requests
+4. Click **Reveal Key** next to your **Secret Key** and copy it.
+
+For detailed API documentation, visit the [Stripe API Docs](https://stripe.com/docs/api).
+
+## Step 3: Retrieve PostgreSQL Credentials
+In this example, we are using a **cloud-hosted PostgreSQL instance**.
+1. Log in to your cloud provider (e.g., [Neon](https://neon.tech/)).
+2. Navigate to your **PostgreSQL instance** and retrieve:
+   - **Host**: Example - `your-db-instance.aws.com`
+   - **Port**: Default is `5432`
+   - **Database Name**: The database where Stripe data will be stored.
+   - **Username & Password**: Credentials used to connect to PostgreSQL.
+   - **sslmode**: Required depending on the security settings of your PostgreSQL instance
+
+## Step 4: Run the `ingestr` Command
+Execute the following command to load data from **Stripe to PostgreSQL**:
+
+```bash
+ingestr ingest \
+    --source-uri 'stripe://?api_key=<YOUR_STRIPE_SECRET_KEY>' \
+    --source-table 'event' \
+    --dest-uri 'postgresql://<username>:<password>@<host>:<port>/<database-name>?sslmode=<sslmode>' \
+    --dest-table 'dest.events'
+```
+- `--source-uri 'stripe://?api_key=<YOUR_STRIPE_SECRET_KEY>'`: Connects to Stripe using secret key .
+- `--source-table 'event'`: Specifies the data to fetch data from Stripe.
+- `--dest-uri 'postgresql://<username>:<password>@<host>:<port>/<database-name>?sslmode=<sslmode>'`: Connects to PostgreSQL as the destination database using the provided credentials.
+- `--dest-table 'dest.events'`: Defines where the data will be stored in Postgres.
+
+## Step 5: Verify Data in PostgreSQL via DBeaver
+
+Once the command runs successfully, let's verify that the Stripe data has been loaded into PostgreSQL.
+
+### 1: Connect to PostgreSQL via DBeaver
+1. **Open DBeaver** and go to **Database** → **New Connection**.
+2. Select **PostgreSQL**.
+3. Enter your **PostgreSQL credentials**:
+   - **Host**: `<your-host>`
+   - **Port**: `5432` (default)
+   - **Database**: `<your-database>`
+   - **Username**: `<your-username>`
+   - **Password**: `<your-password>`
+4. Click **Test Connection** to verify.
+5. If successful, click **Finish**.
+
+### 2: Browse Tables
+1. Expand your **PostgreSQL connection** in the **Database Navigator**.
+2. Expand **Schemas** → **Public**.
+3. Expand **Tables** and find `events`.
+
+### 3: View Data in DBeaver
+1. Right-click the `events` table.
+2. Click **View Data** → **All Rows**.
+3. Ensure that the retrieved data matches what was expected from Stripe.
+
+![Stripe to Postgresql](../media/stripe_postgres.png)
+🎉 Congratulations!
+Your Stripe data is now securely stored in PostgreSQL 🚀

ingestr-0.13.20/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.20"

{ingestr-0.13.19 → ingestr-0.13.20}/ingestr/src/factory.py

@@ -72,6 +72,7 @@ SQL_SOURCE_SCHEMES = [
     "hana",
     "clickhouse",
     "databricks",
+    "db2",
 ]
 
 

{ingestr-0.13.19 → ingestr-0.13.20}/ingestr/src/sources.py

@@ -177,6 +177,9 @@ class SqlSource:
                 scheme="clickhouse+native",
                 query=urlencode(query_params, doseq=True),
             ).geturl()
+         
+        if uri.startswith("db2://"):
+            uri = uri.replace("db2://", "db2+ibm_db://")
 
         query_adapters = []
         if kwargs.get("sql_limit"):

{ingestr-0.13.19 → ingestr-0.13.20}/requirements.in

@@ -47,3 +47,5 @@ clickhouse-connect==0.8.14
 clickhouse-driver==0.2.9
 clickhouse-sqlalchemy==0.2.7
 zstd==1.5.6.5
+ibm-db==3.2.6
+ibm-db-sa==0.4.1

{ingestr-0.13.19 → ingestr-0.13.20}/requirements.txt

@@ -207,6 +207,12 @@ httplib2==0.22.0
     #   google-auth-httplib2
 humanize==4.12.1
     # via dlt
+ibm-db==3.2.6
+    # via
+    #   -r requirements.in
+    #   ibm-db-sa
+ibm-db-sa==0.4.1
+    # via -r requirements.in
 idna==3.10
     # via
     #   requests
@@ -476,6 +482,7 @@ sqlalchemy==1.4.52
     #   databricks-sql-connector
     #   databricks-sqlalchemy
     #   duckdb-engine
+    #   ibm-db-sa
     #   snowflake-sqlalchemy
     #   sqlalchemy-bigquery
     #   sqlalchemy-hana

ingestr-0.13.19/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.19"