ingestr 0.13.18__tar.gz → 0.13.20__tar.gz

{ingestr-0.13.18 → ingestr-0.13.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.18
+Version: 0.13.20
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -81,6 +81,8 @@ Requires-Dist: hdbcli==2.23.27
 Requires-Dist: hexbytes==1.3.0
 Requires-Dist: httplib2==0.22.0
 Requires-Dist: humanize==4.12.1
+Requires-Dist: ibm-db-sa==0.4.1
+Requires-Dist: ibm-db==3.2.6
 Requires-Dist: idna==3.10
 Requires-Dist: inflection==0.5.1
 Requires-Dist: isodate==0.7.2

{ingestr-0.13.18 → ingestr-0.13.20}/docs/.vitepress/config.mjs

@@ -50,6 +50,14 @@ export default defineConfig({
           { text: "example-uris", link: "/commands/example-uris.md" },
         ],
       },
+      {
+        text: "Tutorials",
+        items: [
+          { text: "Load Kinesis Data to BigQuery", link: "/tutorials/load-kinesis-bigquery.md" },
+          { text: "Load Personio Data to DuckDB", link: "/tutorials/load-personio-duckdb.md" },
+          { text: "Load Stripe Data to Postgres", link: "/tutorials/load-stripe-postgres.md" },
+        ],
+      },
       {
         text: "Sources & Destinations",
         items: [
@@ -67,6 +75,7 @@ export default defineConfig({
                 text: "Google BigQuery",
                 link: "/supported-sources/bigquery.md",
               },
+              { text: "IBM Db2", link: "/supported-sources/db2.md" },
               { text: "Kafka", link: "/supported-sources/kafka.md" },
               { text: "Local CSV Files", link: "/supported-sources/csv.md" },
               {

ingestr-0.13.20/docs/supported-sources/db2.md

@@ -0,0 +1,20 @@
+# IBM Db2
+[IBM Db2](https://www.ibm.com/db2) is a high-performance, enterprise-grade relational database system designed for reliability, scalability, and transactional integrity.
+
+ingestr supports IBM Db2 as a source.
+
+## URI format
+The URI format for DB2 is as follows:
+
+```plaintext
+db2://<username>:<password>@<host>:<port>/<database-name>
+```
+
+URI parameters:
+- `username`: The username to connect to the database
+- `password`: The password for the user
+- `host`: The host address of the database server
+- `port`: The port number the database server is listening
+- `database-name`: the name of the database to connect to
+
+The same URI structure can be used both for sources and destinations. You can read more about SQLAlchemy's Db2 dialect [here](https://github.com/ibmdb/python-ibmdbsa?tab=readme-ov-file#connecting).

ingestr-0.13.20/docs/tutorials/load-kinesis-bigquery.md

@@ -0,0 +1,67 @@
+# Load Kinesis Data to BigQuery
+
+Welcome! 👋 This tutorial will guide you through loading data from `Amazon Kinesis` into `Google BigQuery` using `ingestr`, a command-line tool that enables data ingestion between any source and destination using simple flags, no coding required.
+
+Amazon Kinesis is a cloud-based service for real-time data streaming and analytics that processes large data streams in real-time. You often need to store this data in a data warehouse like BigQuery for analysis and reporting. This is where ingestr simplifies the process.
+
+## Prerequisites
+- Install ingestr by following the instructions [here](../getting-started/quickstart.md#Installation)
+- AWS credentials - Access key and Secret key
+- BigQuery service account
+
+## Configuration Steps 
+### Source Configuration - Kinesis
+
+#### `--source-uri`
+This flag connects to your Kinesis stream. The URI format is:
+
+```bash
+kinesis://?aws_access_key_id=$KEY_ID&aws_secret_access_key=$SECRET_KEY&region_name=eu-central-1
+```
+
+Required parameters:
+- `aws_access_key_id`: Your AWS access key
+- `aws_secret_access_key`: Your AWS secret key
+- `region_name`: AWS region of your Kinesis stream.
+
+#### `--source-table`
+This flag specifies which Kinesis stream to read from:
+```bash
+--source-table 'kinesis_stream_name'
+```
+
+### Destination Configuration - BigQuery
+#### `--dest-uri`
+
+This flag connects to BigQuery. The URI format is:
+```bash
+bigquery://project-name?credentials_path=/path/to/service/account.json&location=<location>
+```
+Required parameters:
+- `project-name`: Your BigQuery project name
+- `credentials_path`: Path to service account JSON file
+- `location`: (Optional) Dataset location
+
+#### `--dest-table`
+This flag specifies where to save the data:
+
+```bash
+--dest-table 'schema.table_name'
+```
+
+Now that we've configured all our flags, we can run a single command to connect to Kinesis, read from our specified stream, and load the data into our BigQuery target table.
+
+```bash
+ingestr ingest \
+    --source-uri 'kinesis://?aws_access_key_id=id_123&aws_secret_access_key=secret_123&region_name=eu-central-1' \
+    --source-table 'stream_name_1' \
+    --dest-uri 'bigquery://test-playground?credentials_path=/Users/abc.json' \
+    --dest-table 'dest.results'
+```
+
+After running this command, your Kinesis data will be loaded into BigQuery. Here's what the data looks like in the destination:
+
+<img alt="kinesis_bigquery" src="../media/kinesis.bigquery.png" />
+
+🎉 Congratulations!
+You've successfully loaded data from Amazon Kinesis to your desired destination.

ingestr-0.13.20/docs/tutorials/load-personio-duckdb.md

@@ -0,0 +1,95 @@
+# Load Data from Personio to DuckDB
+
+Welcome! 👋  
+ This beginner-friendly guide will help you load data from `Personio` into `DuckDB` using ingestr — a simple yet powerful command-line tool. No prior experience is needed, and best of all, no coding required!
+
+By the end of this guide, you'll have your Personio data securely stored in DuckDB. But before we dive in, let’s take a quick look at `ingestr`
+
+## Overview of ingestr
+`ingestr` is a command-line tool that simplifies data ingestion by allowing users to load data from a source to a destination using simple command-line flags.
+
+### `ingestr` Command
+```bash
+ingestr ingest \
+   --source-uri '<your-source-uri-here>' \
+   --source-table '<your-schema>.<your-table>' \
+   --dest-uri '<your-destination-uri-here>' \
+   --dest-table '<your-schema>.<your-table>'
+```
+- `ingestr ingest`: Executes the data ingestion process.
+- `--source-uri TEXT`: Specifies the URI of the data source.
+- `--dest-uri TEXT`: Specifies the URI of the destination.
+- `--source-table TEXT`: Defines the table to fetch data from.
+- `--dest-table TEXT`: Specifies the destination table. If not provided, it defaults to `--source-table`.
+
+With this command, we connect to the source, retrieve the specified data, and load it into the destination database.
+## Let's Load Data from Personio to DuckDB Together!
+
+Personio is a human resources management platform that helps businesses handle recruitment and employee data. To analyze this data, you may need to load it into an analytics database like DuckDB. `ingestr` makes this process simple.
+
+### Step 1: Install `ingestr`
+Ensure `ingestr` is installed. If not, follow the installation guide [here](../getting-started/quickstart.md#Installation).
+
+### Step 2: Get Personio Credentials
+Personio will be our data source.
+1. Log in to your Personio account.
+2. Ensure your user has API access.
+3. Navigate to **Settings > Integrations > API Credentials**.
+4. Click **Generate new credentials**.
+5. Assign **read access** to required attributes (e.g., last name, last modified).
+6. Copy the generated `client ID` and `client secret`.
+
+For more details, refer to [Personio API documentation](https://developer.personio.de/docs/getting-started-with-the-personio-api#21-employee-attendance-and-absence-endpoints).
+
+### Step 3: Install DuckDB
+DuckDB will be our data destination. If it’s not already installed, you can install it using pip:
+```bash
+pip install duckdb
+```  
+Alternatively, you can download the latest version from the [official DuckDB website](https://duckdb.org/docs/installation/?version=stable&environment=cli&platform=macos&download_method=direct).
+
+### Step 4: Run the `ingestr` Command
+Execute the following command to load data from Personio to DuckDB:
+```bash
+ingestr ingest \
+    --source-uri 'personio://?client_id=<YOUR_CLIENT_ID>&client_secret=<YOUR_CLIENT_SECRET>' \
+    --source-table 'employees' \
+    --dest-uri 'duckdb:///personio.duckdb' \
+    --dest-table 'dest.employees'
+```
+- `--source-uri 'personio://?client_id=<ID>&client_secret=<SECRET>'`: Connects to Personio using API credentials.
+- `--source-table 'employees'`: Specifies the table to fetch data from Personio.
+- `--dest-uri 'duckdb:///personio.duckdb'`: Specifies DuckDB as the destination database.
+- `--dest-table 'dest.employees'`: Defines where the data will be stored in DuckDB.
+
+
+## Verify Data in DuckDB
+Once the command runs successfully, your Personio data will be available in DuckDB. Follow these steps to verify the data:
+
+### Step 1: Open DuckDB
+If DuckDB is installed, you can open it using the following command:
+```bash
+duckdb personio.duckdb
+```
+
+### Step 2: List Available Tables
+To check if the `employees` table has been created, run:
+```sql
+.tables;
+```
+
+### Step 3: View Data in the `employees` Table
+To display all data from the `employees` table, run:
+```sql
+SELECT * FROM dest.employees;
+```
+
+### Step 4: Validate Data
+Ensure that the retrieved data matches what was expected from Personio.
+
+Example output:
+
+![Personio to DuckDB](../media/personio_duckdb.png)
+
+🎉 **Congratulations!** You have successfully loaded data from Personio to DuckDB using `ingestr`. 
+

ingestr-0.13.20/docs/tutorials/load-stripe-postgres.md

@@ -0,0 +1,101 @@
+# Load Data from Stripe to PostgreSQL
+
+Welcome! 👋 This beginner-friendly guide will help you load data from `Stripe` into `PostgreSQL` using ingestr — a simple yet powerful command-line tool. No prior experience is needed, and best of all, no coding required! 🚀
+
+By the end of this guide, you'll have your Stripe data securely stored in PostgreSQL. But before we dive in, let’s take a quick look at ingestr.
+
+## Overview of `ingestr`
+`ingestr` is a command-line tool that simplifies data ingestion by allowing users to load data from a source to a destination.
+
+### `ingestr` Command
+```bash
+ingestr ingest \
+   --source-uri '<your-source-uri-here>' \
+   --source-table '<your-schema>.<your-table>' \
+   --dest-uri '<your-destination-uri-here>' \
+   --dest-table '<your-schema>.<your-table>'
+```
+- `ingestr ingest`: Executes the data ingestion process.
+- `--source-uri TEXT`: Defines the source database URI.
+- `--dest-uri TEXT`: Defines the destination database URI.
+- `--source-table TEXT`: Specifies the table to fetch data from.
+- `--dest-table TEXT`: Defines where the data should be stored (defaults to `--source-table` if not specified).
+
+With this command, we connect to the source, retrieve the specified data, and load it into the destination database.
+
+## Let's Load Data from Stripe into PostgreSQL Together!
+
+Stripe is a popular payment processing platform used by e-commerce and SaaS applications. If you need to analyze Stripe data, you can load it into PostgreSQL, a reliable and widely used relational database.
+
+## Step 1: Install `ingestr`
+Ensure `ingestr` is installed on your system. If not, follow the installation guide [here](../getting-started/quickstart.md#Installation).
+
+```bash
+pip install ingestr
+```
+
+## Step 2: Retrieve Stripe API Credentials
+Stripe will be our **data source**. To retrieve your API credentials:
+
+1. Log in to your **Stripe Dashboard**: [dashboard.stripe.com](https://dashboard.stripe.com)
+2. Navigate to **Developers** → **API Keys**.
+3. Under **Standard Keys**, find:
+   - **Secret Key**: Required for server-to-server requests
+4. Click **Reveal Key** next to your **Secret Key** and copy it.
+
+For detailed API documentation, visit the [Stripe API Docs](https://stripe.com/docs/api).
+
+## Step 3: Retrieve PostgreSQL Credentials
+In this example, we are using a **cloud-hosted PostgreSQL instance**.
+1. Log in to your cloud provider (e.g., [Neon](https://neon.tech/)).
+2. Navigate to your **PostgreSQL instance** and retrieve:
+   - **Host**: Example - `your-db-instance.aws.com`
+   - **Port**: Default is `5432`
+   - **Database Name**: The database where Stripe data will be stored.
+   - **Username & Password**: Credentials used to connect to PostgreSQL.
+   - **sslmode**: Required depending on the security settings of your PostgreSQL instance
+
+## Step 4: Run the `ingestr` Command
+Execute the following command to load data from **Stripe to PostgreSQL**:
+
+```bash
+ingestr ingest \
+    --source-uri 'stripe://?api_key=<YOUR_STRIPE_SECRET_KEY>' \
+    --source-table 'event' \
+    --dest-uri 'postgresql://<username>:<password>@<host>:<port>/<database-name>?sslmode=<sslmode>' \
+    --dest-table 'dest.events'
+```
+- `--source-uri 'stripe://?api_key=<YOUR_STRIPE_SECRET_KEY>'`: Connects to Stripe using secret key .
+- `--source-table 'event'`: Specifies the data to fetch data from Stripe.
+- `--dest-uri 'postgresql://<username>:<password>@<host>:<port>/<database-name>?sslmode=<sslmode>'`: Connects to PostgreSQL as the destination database using the provided credentials.
+- `--dest-table 'dest.events'`: Defines where the data will be stored in Postgres.
+
+## Step 5: Verify Data in PostgreSQL via DBeaver
+
+Once the command runs successfully, let's verify that the Stripe data has been loaded into PostgreSQL.
+
+### 1: Connect to PostgreSQL via DBeaver
+1. **Open DBeaver** and go to **Database** → **New Connection**.
+2. Select **PostgreSQL**.
+3. Enter your **PostgreSQL credentials**:
+   - **Host**: `<your-host>`
+   - **Port**: `5432` (default)
+   - **Database**: `<your-database>`
+   - **Username**: `<your-username>`
+   - **Password**: `<your-password>`
+4. Click **Test Connection** to verify.
+5. If successful, click **Finish**.
+
+### 2: Browse Tables
+1. Expand your **PostgreSQL connection** in the **Database Navigator**.
+2. Expand **Schemas** → **Public**.
+3. Expand **Tables** and find `events`.
+
+### 3: View Data in DBeaver
+1. Right-click the `events` table.
+2. Click **View Data** → **All Rows**.
+3. Ensure that the retrieved data matches what was expected from Stripe.
+
+![Stripe to Postgresql](../media/stripe_postgres.png)
+🎉 Congratulations!
+Your Stripe data is now securely stored in PostgreSQL 🚀

{ingestr-0.13.18 → ingestr-0.13.20}/ingestr/main.py

@@ -8,6 +8,9 @@ from rich.console import Console
 from rich.status import Status
 from typing_extensions import Annotated
 
+import ingestr.src.partition as partition
+import ingestr.src.resource as resource
+from ingestr.src.destinations import AthenaDestination
 from ingestr.src.filters import cast_set_to_list
 from ingestr.src.telemetry.event import track
 
@@ -357,14 +360,6 @@ def ingest(
             )
             raise typer.Abort()
 
-    def run_on_resource(source, executable):
-        if hasattr(source, "selected_resources") and source.selected_resources:
-            resource_names = list(source.selected_resources.keys())
-            for res in resource_names:
-                executable(source.resources[res])
-        else:
-            executable(source)
-
     def parse_columns(columns: list[str]) -> dict[str, TDataType]:
         from typing import cast, get_args
 
@@ -553,20 +548,23 @@ def ingest(
             sql_exclude_columns=sql_exclude_columns,
         )
 
-        run_on_resource(dlt_source, lambda x: x.add_map(cast_set_to_list))
+        resource.for_each(dlt_source, lambda x: x.add_map(cast_set_to_list))
 
         def col_h(x):
             if column_hints:
                 x.apply_hints(columns=column_hints)
 
-        run_on_resource(dlt_source, col_h)
+        resource.for_each(dlt_source, col_h)
+
+        if isinstance(destination, AthenaDestination) and partition_by:
+            partition.apply_athena_hints(dlt_source, partition_by, column_hints)
 
         if original_incremental_strategy == IncrementalStrategy.delete_insert:
 
             def set_primary_key(x):
                 x.incremental.primary_key = ()
 
-            run_on_resource(dlt_source, set_primary_key)
+            resource.for_each(dlt_source, set_primary_key)
 
         if (
             factory.destination_scheme in PARQUET_SUPPORTED_DESTINATIONS

{ingestr-0.13.18 → ingestr-0.13.20}/ingestr/src/adjust/adjust_helpers.py

@@ -82,7 +82,7 @@ class AdjustAPI:
             items = result.get("rows", [])
             yield items
         else:
-            raise HTTPError(f"Request failed with status code: {response.status_code}")
+            raise HTTPError(f"Request failed with status code: {response.status_code}, {response.text}.")
 
     def fetch_events(self):
         headers = {"Authorization": f"Bearer {self.api_key}"}
@@ -93,7 +93,7 @@ class AdjustAPI:
             result = response.json()
             yield result
         else:
-            raise HTTPError(f"Request failed with status code: {response.status_code}")
+            raise HTTPError(f"Request failed with status code: {response.status_code}, {response.text}.")
 
 
 def parse_filters(filters_raw: str) -> dict:

{ingestr-0.13.18 → ingestr-0.13.20}/ingestr/src/blob.py

@@ -1,6 +1,6 @@
 import warnings
 from typing import Tuple, TypeAlias
-from urllib.parse import ParseResult
+from urllib.parse import ParseResult, urlparse
 
 BucketName: TypeAlias = str
 FileGlob: TypeAlias = str
@@ -14,13 +14,16 @@ def parse_uri(uri: ParseResult, table: str) -> Tuple[BucketName, FileGlob]:
     Supports the following Forms:
     - uri: "gs://"
       table: "bucket-name/file-glob"
+    - uri: "gs://uri-bucket-name" (uri-bucket-name is preferred)
+      table: "gs://table-bucket-name/file-glob"
+    - uri: "gs://"
+      table: "gs://bucket-name/file-glob"
     - uri: gs://bucket-name/file-glob
       table: None
     - uri: "gs://bucket-name"
       table: "file-glob"
 
-    The first form is the prefered method. Other forms are supported
-    for backward compatibility, but discouraged.
+    The first form is the prefered method. Other forms are supported but discouraged.
     """
 
     table = table.strip()
@@ -34,15 +37,15 @@ def parse_uri(uri: ParseResult, table: str) -> Tuple[BucketName, FileGlob]:
         )
         return host, uri.path.lstrip("/")
 
+    table_uri = urlparse(table)
+
     if host != "":
-        warnings.warn(
-            f"Using the form '{uri.scheme}://bucket-name' is deprecated and will be removed in future versions.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        return host, table.lstrip("/")
+        return host, table_uri.path.lstrip("/")
+
+    if table_uri.hostname:
+        return table_uri.hostname, table_uri.path.lstrip("/")
 
-    parts = table.lstrip("/").split("/", maxsplit=1)
+    parts = table_uri.path.lstrip("/").split("/", maxsplit=1)
     if len(parts) != 2:
         return "", parts[0]
 

ingestr-0.13.20/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.20"

{ingestr-0.13.18 → ingestr-0.13.20}/ingestr/src/factory.py

@@ -72,6 +72,7 @@ SQL_SOURCE_SCHEMES = [
     "hana",
     "clickhouse",
     "databricks",
+    "db2",
 ]
 
 

{ingestr-0.13.18 → ingestr-0.13.20}/ingestr/src/kinesis/__init__.py

@@ -16,6 +16,7 @@ from .helpers import get_shard_iterator, max_sequence_by_shard
     name=lambda args: args["stream_name"],
     primary_key="kinesis_msg_id",
     standalone=True,
+    max_table_nesting=0
 )
 def kinesis_stream(
     stream_name: str,

ingestr-0.13.20/ingestr/src/partition.py

@@ -0,0 +1,31 @@
+from typing import Dict
+
+from dlt.common.schema.typing import TColumnSchema
+from dlt.destinations.adapters import athena_adapter, athena_partition
+from dlt.sources import DltResource, DltSource
+
+import ingestr.src.resource as resource
+
+
+def apply_athena_hints(
+    source: DltSource | DltResource,
+    partition_column: str,
+    additional_hints: Dict[str, TColumnSchema] = {},
+) -> None:
+    def _apply_partition_hint(resource: DltResource) -> None:
+
+        columns = resource.columns if resource.columns else {}
+
+        partition_hint = (
+            columns.get(partition_column)  # type: ignore
+            or additional_hints.get(partition_column)
+        )
+
+        athena_adapter(
+            resource,
+            athena_partition.day(partition_column)
+            if partition_hint and partition_hint.get("data_type") in ("timestamp", "date")
+            else partition_column,
+        )
+
+    resource.for_each(source, _apply_partition_hint)

ingestr-0.13.20/ingestr/src/resource.py

@@ -0,0 +1,17 @@
+from typing import Callable
+
+from dlt.sources import DltResource, DltSource
+
+
+def for_each(
+    source: DltSource | DltResource, ex: Callable[[DltResource], None | DltResource]
+):
+    """
+    Apply a function to each resource in a source.
+    """
+    if hasattr(source, "selected_resources") and source.selected_resources:
+        resource_names = list(source.selected_resources.keys())
+        for res in resource_names:
+            ex(source.resources[res])  # type: ignore[union-attr]
+    else:
+        ex(source)  # type: ignore[arg-type]

{ingestr-0.13.18 → ingestr-0.13.20}/ingestr/src/sources.py

@@ -177,6 +177,9 @@ class SqlSource:
                 scheme="clickhouse+native",
                 query=urlencode(query_params, doseq=True),
             ).geturl()
+         
+        if uri.startswith("db2://"):
+            uri = uri.replace("db2://", "db2+ibm_db://")
 
         query_adapters = []
         if kwargs.get("sql_limit"):

{ingestr-0.13.18 → ingestr-0.13.20}/requirements.in

@@ -47,3 +47,5 @@ clickhouse-connect==0.8.14
 clickhouse-driver==0.2.9
 clickhouse-sqlalchemy==0.2.7
 zstd==1.5.6.5
+ibm-db==3.2.6
+ibm-db-sa==0.4.1

{ingestr-0.13.18 → ingestr-0.13.20}/requirements.txt

@@ -207,6 +207,12 @@ httplib2==0.22.0
     #   google-auth-httplib2
 humanize==4.12.1
     # via dlt
+ibm-db==3.2.6
+    # via
+    #   -r requirements.in
+    #   ibm-db-sa
+ibm-db-sa==0.4.1
+    # via -r requirements.in
 idna==3.10
     # via
     #   requests
@@ -476,6 +482,7 @@ sqlalchemy==1.4.52
     #   databricks-sql-connector
     #   databricks-sqlalchemy
     #   duckdb-engine
+    #   ibm-db-sa
     #   snowflake-sqlalchemy
     #   sqlalchemy-bigquery
     #   sqlalchemy-hana

ingestr-0.13.18/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.18"