ingestr 0.10.2__tar.gz → 0.10.4__tar.gz

ingestr-0.10.4/.githooks/pre-commit-hook.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+echo "scanning for secrets ..."
+
+WORK_DIR="/root/code"
+
+secret_detected() {
+    echo "secrets detected in source code. commit aborted."
+    exit 1
+}
+
+# use gitleaks binary if available
+# else fallback to using docker for running gitleaks
+CMD="gitleaks dir -v"
+
+if [[ ! `which gitleaks`  ]]; then
+    which docker > /dev/null || (echo "gitleaks or docker is required for running secrets scan." && exit 1)
+    CMD="docker run -v $PWD:$WORK_DIR -w $WORK_DIR ghcr.io/gitleaks/gitleaks:latest dir -v"
+fi
+
+$CMD || secret_detected

ingestr-0.10.4/.github/workflows/secrets-scan.yml

@@ -0,0 +1,12 @@
+name: secrets_scan
+on: [pull_request, push, workflow_dispatch]
+jobs:
+  scan:
+    name: gitleaks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: scan for secrets (gitleaks)
+        run: docker run -v $PWD:/code -w /code ghcr.io/gitleaks/gitleaks:latest dir -v

ingestr-0.10.4/.gitleaksignore

@@ -0,0 +1,3 @@
+ingestr/src/telemetry/event.py:generic-api-key:9
+ingestr/src/testdata/fakebqcredentials.json:private-key:5
+docs/supported-sources/shopify.md:generic-api-key:26

{ingestr-0.10.2 → ingestr-0.10.4}/Makefile

@@ -1,11 +1,11 @@
 .ONESHELL:
-.PHONY: test lint format ftl test-ci lint-ci build upload-release
+.PHONY: test lint format test-ci lint-ci build upload-release setup
 
 venv: venv/touchfile
 
 venv/touchfile: requirements-dev.txt requirements.txt
 	test -d venv || python3 -m venv venv
-	. venv/bin/activate; $(MAKE) deps
+	. venv/bin/activate; pip install uv; $(MAKE) deps
 	touch venv/touchfile
 
 deps:
@@ -40,3 +40,7 @@ build:
 
 upload-release:
 	twine upload --verbose dist/*
+
+setup:
+	@echo "installing git hooks ..."
+	@install -m 755 .githooks/pre-commit-hook.sh .git/hooks/pre-commit

{ingestr-0.10.2 → ingestr-0.10.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.10.2
+Version: 0.10.4
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -74,11 +74,20 @@ ingestr is a command-line app that allows you to ingest data from any source int
 ingestr takes away the complexity of managing any backend or writing any code for ingesting data, simply run the command and watch the data land on its destination.
 
 ## Installation
+We recommend using [uv](https://github.com/astral-sh/uv) to run `ingestr`.
 
 ```
-pip install ingestr
+pip install uv
+uvx ingestr
 ```
 
+Alternatively, if you'd like to install it globally:
+```
+uv pip install --system ingestr
+```
+
+While installation with vanilla `pip` is possible, it's an order of magnitude slower.
+
 ## Quickstart
 
 ```bash
@@ -104,6 +113,13 @@ You can see the full documentation [here](https://bruin-data.github.io/ingestr/g
 
 Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shared_invite/zt-2dl2i8foy-bVsuMUauHeN9M2laVm3ZVg).
 
+## Contributing
+
+Pull requests are welcome. However, please open an issue first to discuss what you would like to change. We maybe able to offer you help and feedback regarding any changes you would like to make.
+
+> [!NOTE]
+> After cloning `ingestr` make sure to run `make setup` to install githooks.
+
 ## Supported sources & destinations
 
 <table>

{ingestr-0.10.2 → ingestr-0.10.4}/README.md

@@ -21,11 +21,20 @@ ingestr is a command-line app that allows you to ingest data from any source int
 ingestr takes away the complexity of managing any backend or writing any code for ingesting data, simply run the command and watch the data land on its destination.
 
 ## Installation
+We recommend using [uv](https://github.com/astral-sh/uv) to run `ingestr`.
 
 ```
-pip install ingestr
+pip install uv
+uvx ingestr
 ```
 
+Alternatively, if you'd like to install it globally:
+```
+uv pip install --system ingestr
+```
+
+While installation with vanilla `pip` is possible, it's an order of magnitude slower.
+
 ## Quickstart
 
 ```bash
@@ -51,6 +60,13 @@ You can see the full documentation [here](https://bruin-data.github.io/ingestr/g
 
 Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shared_invite/zt-2dl2i8foy-bVsuMUauHeN9M2laVm3ZVg).
 
+## Contributing
+
+Pull requests are welcome. However, please open an issue first to discuss what you would like to change. We maybe able to offer you help and feedback regarding any changes you would like to make.
+
+> [!NOTE]
+> After cloning `ingestr` make sure to run `make setup` to install githooks.
+
 ## Supported sources & destinations
 
 <table>

{ingestr-0.10.2 → ingestr-0.10.4}/docs/.vitepress/config.mjs

@@ -57,7 +57,7 @@ export default defineConfig({
             text: "Databases",
             collapsed: false,
             items: [
-              { text: "Athena", link: "/supported-sources/athena.md" },
+              { text: "AWS Athena", link: "/supported-sources/athena.md" },
               { text: "AWS Redshift", link: "/supported-sources/redshift.md" },
               { text: "Databricks", link: "/supported-sources/databricks.md" },
               { text: "DuckDB", link: "/supported-sources/duckdb.md" },

{ingestr-0.10.2 → ingestr-0.10.4}/docs/getting-started/quickstart.md

@@ -13,9 +13,19 @@ ingestr takes away the complexity of managing any backend or writing any code fo
 
 
 ## Installation
+We recommend using [uv](https://github.com/astral-sh/uv) to run `ingestr`.
+
+```
+pip install uv
+uvx ingestr
 ```
-pip install ingestr
+
+Alternatively, if you'd like to install it globally:
 ```
+uv pip install --system ingestr
+```
+
+While installation with vanilla `pip` is possible, it's an order of magnitude slower.
 
 ## Quickstart
 

{ingestr-0.10.2 → ingestr-0.10.4}/docs/supported-sources/adjust.md

@@ -62,7 +62,7 @@ ingestr ingest \
     --source-uri 'adjust://?api_key=nr_123' \
     --source-table 'campaigns' \
     --dest-uri duckdb:///adjust.duckdb \
-    --dest-table 'adjust.output'
+    --dest-table 'dest.output'
 ```
 
 Copy creatives data from Adjust into a DuckDB database:
@@ -71,7 +71,7 @@ ingestr ingest \
     --source-uri 'adjust://?api_key=nr_123' \
     --source-table 'creatives' \
     --dest-uri duckdb:///adjust.duckdb \
-    --dest-table 'adjust.output'
+    --dest-table 'dest.output'
 ```
 
 Copy custom data from Adjust into a DuckDB database:

{ingestr-0.10.2 → ingestr-0.10.4}/docs/supported-sources/athena.md

@@ -1,4 +1,4 @@
-# Athena
+# AWS Athena
 [Athena](https://aws.amazon.com/athena/) is an interactive query service that allows users to analyze data directly in Amazon S3 using standard SQL.
 
 The Athena destination stores data as Parquet files in S3 buckets and creates external tables in AWS Glue Catalog.

{ingestr-0.10.2 → ingestr-0.10.4}/docs/supported-sources/gorgias.md

@@ -23,7 +23,7 @@ The URI is used to connect to the Gorgias API for extracting data.
 ingestr ingest --source-table 'tickets' --source-uri $GORGIAS_URI --dest-uri $BIGQUERY_URI --interval-start 2024-06-19  --dest-table 'gorgias.ticket_messages' --loader-file-format jsonl
 
 # get all the customers and write them to `gorgias.customers` table on DuckDB
-ingestr ingest --source-table 'customers' --source-uri $GORGIAS_URI --dest-uri duckdb:///gorgias.duckdb --interval-start 2024-01-01  --dest-table 'gorgias.customers'
+ingestr ingest --source-table 'customers' --source-uri $GORGIAS_URI --dest-uri duckdb:///gorgias.duckdb --interval-start 2024-01-01  --dest-table 'dest.customers'
 ```
 
 ## Supported entities

{ingestr-0.10.2 → ingestr-0.10.4}/docs/supported-sources/gsheets.md

@@ -37,7 +37,7 @@ Once you complete the guide, you should have a service account JSON file and the
 Based on this assumption, here's a sample command that will copy the data from the Google Sheets spreadsheet into a DuckDB database:
 
 ```sh
-ingestr ingest --source-uri 'gsheets://?credentials_path=/path/to/file.json' --source-table 'fkdUQ2bjdNfUq2CA.Sheet1' --dest-uri duckdb:///gsheets.duckdb --dest-table 'gsheets.output'
+ingestr ingest --source-uri 'gsheets://?credentials_path=/path/to/file.json' --source-table 'fkdUQ2bjdNfUq2CA.Sheet1' --dest-uri duckdb:///gsheets.duckdb --dest-table 'dest.output'
 ```
 
 The result of this command will be a table in the `gsheets.duckdb` database.

{ingestr-0.10.2 → ingestr-0.10.4}/docs/supported-sources/kafka.md

@@ -32,7 +32,7 @@ ingestr ingest \
     --source-uri 'kafka://?bootstrap_servers=localhost:9092&group_id=test_group' \
     --source-table 'my-topic' \
     --dest-uri duckdb:///kafka.duckdb \
-    --dest-table 'kafka.my_topic'
+    --dest-table 'dest.my_topic'
 ```
 
 The result of this command will be a table in the `kafka.duckdb` database with JSON columns.

{ingestr-0.10.2 → ingestr-0.10.4}/docs/supported-sources/klaviyo.md

@@ -19,10 +19,10 @@ URI parameters:
 The URI is used to connect to the Klaviyo API for extracting data.
 
 ```bash
-ingestr ingest --source-table 'events' --source-uri 'klaviyo://?api_key=pk_test' --dest-uri duckdb:///klaviyo.duckdb --interval-start 2022-01-01 --dest-table 'klaviyo.events' --extract-parallelism 20
+ingestr ingest --source-table 'events' --source-uri 'klaviyo://?api_key=pk_test' --dest-uri duckdb:///klaviyo.duckdb --interval-start 2022-01-01 --dest-table 'dest.events' --extract-parallelism 20
 ```
 
-This command fetches all the events that are created/updated since 2022-01-01 and writes them to `klaviyo.events` table on DuckDB, using 20 parallel threads to improve performance and efficiently handle large data .
+This command fetches all the events that are created/updated since 2022-01-01 and writes them to `dest.events` table on DuckDB, using 20 parallel threads to improve performance and efficiently handle large data .
 
 ## Tables
 

{ingestr-0.10.2 → ingestr-0.10.4}/docs/supported-sources/notion.md

@@ -22,7 +22,7 @@ Notion requires a few steps to set up an integration, please follow the guide dl
 Once you complete the guide, you should have an API key, and the table ID to connect to. Let's say your API token is `secret_12345` and the database you'd like to connect to is `bfeaafc0c25f40a9asdasd672a9456f3`, here's a sample command that will copy the data from the Notion table into a DuckDB database:
 
 ```sh
-ingestr ingest --source-uri 'notion://?api_key=secret_12345' --source-table 'bfeaafc0c25f40a9asdasd672a9456f3' --dest-uri duckdb:///notion.duckdb --dest-table 'notion.output'
+ingestr ingest --source-uri 'notion://?api_key=secret_12345' --source-table 'bfeaafc0c25f40a9asdasd672a9456f3' --dest-uri duckdb:///notion.duckdb --dest-table 'dest.output'
 ```
 
 The result of this command will be a table in the `notion.duckdb` database with JSON columns. 

{ingestr-0.10.2 → ingestr-0.10.4}/docs/supported-sources/shopify.md

@@ -23,7 +23,7 @@ Shopify requires a few steps to set up an integration, please follow the guide d
 Once you complete the guide, you should have an API key and the store name to connect to. Let's say your API key is `shpkey_12345` and the store you'd like to connect to is `my-store`, here's a sample command that will copy the data from the Shopify store into a DuckDB database:
 
 ```sh
-ingestr ingest --source-uri 'shopify://my-store.myshopify.com?api_key=shpkey_12345' --source-table 'orders' --dest-uri duckdb:///shopify.duckdb --dest-table 'shopify.orders'
+ingestr ingest --source-uri 'shopify://my-store.myshopify.com?api_key=shpkey_12345' --source-table 'orders' --dest-uri duckdb:///shopify.duckdb --dest-table 'dest.orders'
 ```
 
 The result of this command will be a table in the `shopify.duckdb` database with JSON columns.

{ingestr-0.10.2 → ingestr-0.10.4}/docs/supported-sources/stripe.md

@@ -25,7 +25,7 @@ Stripe requires a few steps to set up an integration, please follow the guide dl
 Once you complete the guide, you should have an API key. Let's say your API key is `sk_test_12345`, here's a sample command that will copy the data from Stripe into a DuckDB database:
 
 ```sh
-ingestr ingest --source-uri 'stripe://?api_key=sk_test_12345' --source-table 'charges' --dest-uri duckdb:///stripe.duckdb --dest-table 'stripe.charges'
+ingestr ingest --source-uri 'stripe://?api_key=sk_test_12345' --source-table 'charges' --dest-uri duckdb:///stripe.duckdb --dest-table 'dest.charges'
 ```
 
 The result of this command will be a table in the `stripe.duckdb` database with JSON columns.

{ingestr-0.10.2 → ingestr-0.10.4}/docs/supported-sources/zendesk.md

@@ -39,7 +39,7 @@ Once you complete the guide, if you decide to use an OAuth token, you should hav
 ingestr ingest --source-uri "zendesk://:qVsbdiasVt@mycompany" \
 --source-table 'tickets' \
 --dest-uri 'duckdb:///zendesk.duckdb' \
---dest-table 'zendesk.tickets' \
+--dest-table 'dest.tickets' \
 --interval-start '2024-01-01'
 ```
 
@@ -49,7 +49,7 @@ If you decide to use an API Token, you should have a subdomain, email, and API t
 ingestr ingest --source-uri "zendesk://john@get.com:nbs123@mycompany" \
 --source-table 'tickets' \
 --dest-uri 'duckdb:///zendesk.duckdb' \
---dest-table 'zendesk.tickets' \
+--dest-table 'dest.tickets' \
 --interval-start '2024-01-01'
 ```
 

{ingestr-0.10.2 → ingestr-0.10.4}/ingestr/main.py

@@ -288,6 +288,7 @@ def ingest(
         ),
     ] = [],  # type: ignore
 ):
+    # TODO(turtledev): can't we move this to the top of this file?
     import hashlib
     import tempfile
     from datetime import datetime
@@ -383,6 +384,15 @@ def ingest(
         )
 
         factory = SourceDestinationFactory(source_uri, dest_uri)
+        track(
+            "command_running",
+            {
+                "command": "ingest",
+                "source_type": factory.source_scheme,
+                "destination_type": factory.destination_scheme,
+            },
+        )
+
         source = factory.get_source()
         destination = factory.get_destination()
 

{ingestr-0.10.2 → ingestr-0.10.4}/ingestr/src/adjust/__init__.py

@@ -67,13 +67,16 @@ def adjust_source(
             filters=filters,
         )
 
+    if not dimensions:
+        return campaigns, creatives
+
     merge_key = merge_key
+    type_hints = {}
     for dimension in REQUIRED_CUSTOM_DIMENSIONS:
         if dimension in dimensions:
             merge_key = dimension
             break
 
-    type_hints = {}
     for dimension in dimensions:
         if dimension in KNOWN_TYPE_HINTS:
             type_hints[dimension] = KNOWN_TYPE_HINTS[dimension]

{ingestr-0.10.2 → ingestr-0.10.4}/ingestr/src/chess/__init__.py

@@ -12,7 +12,7 @@ from .helpers import get_path_with_retry, get_url_with_retry, validate_month_str
 from .settings import UNOFFICIAL_CHESS_API_URL
 
 
-@dlt.source(name="chess")
+@dlt.source(name="chess", max_table_nesting=0)
 def source(
     players: List[str], start_month: str = None, end_month: str = None
 ) -> Sequence[DltResource]:

{ingestr-0.10.2 → ingestr-0.10.4}/ingestr/src/destinations.py

@@ -250,7 +250,14 @@ class AthenaDestination:
         )
 
     def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:
-        return {}
+        table_fields = table.split(".")
+        if len(table_fields) != 2:
+            raise ValueError("Table name must be in the format <schema>.<table>")
+        return {
+            "table_format": "iceberg",
+            "dataset_name": table_fields[-2],
+            "table_name": table_fields[-1],
+        }
 
     def post_load(self):
         pass

{ingestr-0.10.2 → ingestr-0.10.4}/ingestr/src/hubspot/__init__.py

@@ -50,7 +50,7 @@ from .settings import (
 THubspotObjectType = Literal["company", "contact", "deal", "ticket", "product", "quote"]
 
 
-@dlt.source(name="hubspot")
+@dlt.source(name="hubspot", max_table_nesting=0)
 def hubspot(
     api_key: str = dlt.secrets.value,
     include_history: bool = False,

{ingestr-0.10.2 → ingestr-0.10.4}/ingestr/src/sources.py

@@ -622,7 +622,7 @@ class HubspotSource:
 
 class AirtableSource:
     def handles_incrementality(self) -> bool:
-        return True
+        return False
 
     # airtable://?access_token=<access_token>&base_id=<base_id>
 

{ingestr-0.10.2 → ingestr-0.10.4}/ingestr/src/stripe_analytics/__init__.py

@@ -11,7 +11,7 @@ from .helpers import pagination, transform_date
 from .settings import ENDPOINTS, INCREMENTAL_ENDPOINTS
 
 
-@dlt.source
+@dlt.source(max_table_nesting=0)
 def stripe_source(
     endpoints: Tuple[str, ...] = ENDPOINTS,
     stripe_secret_key: str = dlt.secrets.value,

ingestr-0.10.4/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.10.4"

{ingestr-0.10.2 → ingestr-0.10.4}/ingestr/src/zendesk/__init__.py

@@ -23,7 +23,6 @@ from .settings import (
     TALK_ENDPOINTS,
 )
 
-
 @dlt.source(max_table_nesting=0)
 def zendesk_talk(
     credentials: TZendeskCredentials = dlt.secrets.value,

ingestr-0.10.2/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.10.2"