ingestr 0.10.3__tar.gz → 0.12.1__tar.gz

{ingestr-0.10.3 → ingestr-0.12.1}/Makefile

@@ -1,5 +1,5 @@
 .ONESHELL:
-.PHONY: test lint format test-ci lint-ci build upload-release setup
+.PHONY: test lint format test-ci lint-ci build upload-release setup docker-shell
 
 venv: venv/touchfile
 
@@ -43,4 +43,8 @@ upload-release:
 
 setup:
 	@echo "installing git hooks ..."
-	@install -m 755 .githooks/pre-commit-hook.sh .git/hooks/pre-commit
+	@install -m 755 .githooks/pre-commit-hook.sh .git/hooks/pre-commit
+
+docker-shell:
+	# run a docker container to build and run ingestr
+	@docker run -v $(PWD):/root/code -w /root/code -it --rm --entrypoint /bin/bash python:3.11

{ingestr-0.10.3 → ingestr-0.12.1}/PKG-INFO

@@ -1,10 +1,11 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: ingestr
-Version: 0.10.3
+Version: 0.12.1
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
 Author-email: Burak Karakan <burak.karakan@getbruin.com>
+License-File: LICENSE.md
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
@@ -13,6 +14,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Database
 Requires-Python: >=3.9
+Requires-Dist: asana==3.2.3
 Requires-Dist: confluent-kafka>=2.6.1
 Requires-Dist: databricks-sql-connector==2.9.3
 Requires-Dist: dlt==1.4.0
@@ -199,7 +201,7 @@ Pull requests are welcome. However, please open an issue first to discuss what y
     <tr>
         <td colspan="3" style='text-align:center;'><strong>Platforms</strong></td>
     </tr>
-    <td>Adjust</td>
+        <td>Adjust</td>
         <td>✅</td>
         <td>-</td>
     <tr>
@@ -207,17 +209,27 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
-     <tr>
+    <tr>
         <td>AppsFlyer</td>
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Asana</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Chess.com</td>
         <td>✅</td>
         <td>-</td>
     </tr>
-     <tr>
+    <tr>
+        <td>DynamoDB</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+    <tr>
         <td>Facebook Ads</td>
         <td>✅</td>
         <td>-</td>

{ingestr-0.10.3 → ingestr-0.12.1}/README.md

@@ -146,7 +146,7 @@ Pull requests are welcome. However, please open an issue first to discuss what y
     <tr>
         <td colspan="3" style='text-align:center;'><strong>Platforms</strong></td>
     </tr>
-    <td>Adjust</td>
+        <td>Adjust</td>
         <td>✅</td>
         <td>-</td>
     <tr>
@@ -154,17 +154,27 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
-     <tr>
+    <tr>
         <td>AppsFlyer</td>
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Asana</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Chess.com</td>
         <td>✅</td>
         <td>-</td>
     </tr>
-     <tr>
+    <tr>
+        <td>DynamoDB</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+    <tr>
         <td>Facebook Ads</td>
         <td>✅</td>
         <td>-</td>

{ingestr-0.10.3 → ingestr-0.12.1}/docs/.vitepress/config.mjs

@@ -57,7 +57,7 @@ export default defineConfig({
             text: "Databases",
             collapsed: false,
             items: [
-              { text: "Athena", link: "/supported-sources/athena.md" },
+              { text: "AWS Athena", link: "/supported-sources/athena.md" },
               { text: "AWS Redshift", link: "/supported-sources/redshift.md" },
               { text: "Databricks", link: "/supported-sources/databricks.md" },
               { text: "DuckDB", link: "/supported-sources/duckdb.md" },
@@ -88,7 +88,9 @@ export default defineConfig({
               { text: "Adjust", link: "/supported-sources/adjust.md" },
               { text: "Airtable", link: "/supported-sources/airtable.md" },
               { text: "AppsFlyer", link: "/supported-sources/appsflyer.md" },
+              { text: "Asana", link: "/supported-sources/asana.md" },
               { text: "Chess.com", link: "/supported-sources/chess.md" },
+              { text: "DynamoDB", link: "/supported-sources/dynamodb.md" },
               {
                 text: "Facebook Ads",
                 link: "/supported-sources/facebook-ads.md",

{ingestr-0.10.3 → ingestr-0.12.1}/docs/supported-sources/adjust.md

@@ -62,7 +62,7 @@ ingestr ingest \
     --source-uri 'adjust://?api_key=nr_123' \
     --source-table 'campaigns' \
     --dest-uri duckdb:///adjust.duckdb \
-    --dest-table 'adjust.output'
+    --dest-table 'dest.output'
 ```
 
 Copy creatives data from Adjust into a DuckDB database:
@@ -71,7 +71,7 @@ ingestr ingest \
     --source-uri 'adjust://?api_key=nr_123' \
     --source-table 'creatives' \
     --dest-uri duckdb:///adjust.duckdb \
-    --dest-table 'adjust.output'
+    --dest-table 'dest.output'
 ```
 
 Copy custom data from Adjust into a DuckDB database:

ingestr-0.12.1/docs/supported-sources/asana.md

@@ -0,0 +1,58 @@
+# Asana
+[Asana](https://asana.com/) is a software-as-a-service platform designed for team collaboration and work management. Teams can create projects, assign tasks, set deadlines, and communicate directly within Asana. It also includes reporting tools, file attachments, calendars, and goal tracking.
+
+## URI format
+
+The URI format for Asana is as follows:
+```
+asana://<workspace_id>?access_token=<access_token>
+```
+
+URI parameters:
+- `workspace_id` is the `gid` of the workspace. 
+- `access_token` is a personal access token.
+
+You can obtain `workspace_id` by going to the [admin console](https://help.asana.com/s/article/how-to-access-the-admin-console). The URL in your browser will look something like this:
+
+```
+https://app.asana.com/admin/fake-123456789/
+```
+
+In this example `fake-123456789` is your workspace id.
+
+## Setting up an Asana Integration
+
+You can obtain a personal access token from the [developer console](https://app.asana.com/0/my-apps). For more information, see [Asana developers documentation](https://developers.asana.com/docs/personal-access-token).
+
+## Example
+Let's say you have a workspace with id `workspace-1337` and you want to ingest all tasks into a duckdb database called `work.db`. For this example the value of `access_token` will be `fake_token`
+
+You can run the following to achieve this:
+```sh
+ingestr ingest \
+  --source-uri "asana://workspace-1337?access_token=fake_token" \
+  --source-table "tasks" \
+  --dest-uri "duckdb://./work.db" \
+  --dest-table "public.tasks"
+```
+
+
+## Tables
+
+Asana source allows ingesting the following sources into separate tables:
+
+| **Table**    | **Description**                                                                 |
+|---------------|---------------------------------------------------------------------------------|
+| `workspaces`  | Information about people, materials, or assets required to complete a task or project successfully. |
+| `projects`    | Collections of tasks and related information.                                   |
+| `tasks`    | Tasks within a project. Only tasks that belong to a project can be ingested. Users private tasks are not ingested, for example.      |
+| `projects`    | Collections of tasks and related information.                                   |
+| `tags`        | Labels that can be attached to tasks, projects, or conversations to help categorize and organize them. |
+| `stories`     | Updates or comments that team members can add to a task or project.             |
+| `teams`       | Groups of individuals who work together to complete projects and tasks.         |
+| `users`       | Individuals who have access to the Asana platform.                              |
+
+Use these as `--source-table` parameter in the `ingestr ingest` command.
+
+> [!WARNING]
+> Asana does not support incremental loading for many endpoints in its APIs, which means ingestr will load endpoints incrementally if they support it, and do a full-refresh if not.

{ingestr-0.10.3 → ingestr-0.12.1}/docs/supported-sources/athena.md

@@ -1,4 +1,4 @@
-# Athena
+# AWS Athena
 [Athena](https://aws.amazon.com/athena/) is an interactive query service that allows users to analyze data directly in Amazon S3 using standard SQL.
 
 The Athena destination stores data as Parquet files in S3 buckets and creates external tables in AWS Glue Catalog.

ingestr-0.12.1/docs/supported-sources/dynamodb.md

@@ -0,0 +1,90 @@
+# DynamoDB
+
+Amazon [DynamoDB](https://aws.amazon.com/dynamodb/) is a managed NoSQL database service provided by Amazon Web Services (AWS). It supports key-value and document data structures and is designed to handle a wide range of applications requiring scalability and performance. 
+
+## URI format
+
+The URI format for DynamoDB is as follows:
+```plaintext
+dynamodb://dynamodb.<region>.amazonaws.com?access_key_id=<aws_access_key_id>&secret_access_key=<aws_secret_access_key>
+```
+
+URI parameters:
+
+- `access_key_id`: Identifes an IAM account. 
+- `secret_access_key`: Password for the IAM account.
+
+
+## Setting up a DynamoDB integration
+
+### Prerequisites
+* AWS IAM access key pair.
+* A DynamoDB Table that you will to load data from
+
+To obtain the access keys, use the IAM console on AWS. See [IAM Documentation](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) for more information.
+
+
+### Configuring Permissions
+To use DynamoDB source, the user account must have the following IAM permissions:
+* `dynamodb:DescribeTable`
+* `dynamodb:Scan`
+
+Following AWS Best practices, you can create an IAM policy that you can assign to the user account you wish to use with `ingestr`.
+Below is a sample policy:
+```json
+{
+	"Version": "2012-10-17",
+	"Statement": [
+		{
+			"Sid": "Statement1",
+			"Effect": "Allow",
+			"Action": [
+				"dynamodb:DescribeTable",
+				"dynamodb:Scan"
+			],
+			"Resource": [
+				"<TABLE_ARN>"
+			]
+		}
+	]
+}
+```
+
+Replace `TABLE_ARN` with the DynamoDB [Amazon Resource Name](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference-arns.html) of your Table. You can find the ARN for your table in the [DynamoDB console](https://console.aws.amazon.com/dynamodb/home). You can add as many tables you want in the `Resource` field. Alternatively, if you'd like to give access to all tables that you own, you can set `Resource` to `["*"]`.
+
+### Example: Simple load
+
+For this example, we'll assume the value of `access_key_id` and `secret_access_key` are `user` and `pass` respectively.
+
+Say you have a table called `absolute-armadillo` in the region `ap-south-1` and you want to load this data to a duckdb database called `animal.db`.
+
+You run the following to achieve this:
+```sh
+ingestr ingest \
+    --source-uri "dynamodb://dynamodb.ap-south-1.amazonaws.com?access_key_id=user&secret_access_key=pass" \
+    --source-table "absolute-armadillo" \
+    --dest-uri "duckdb://./animal.db" \
+    --dest-table "public.armadillo"
+```
+
+### Example: Incremental load
+`ingestr` supports incremental loading. Incremental loading is a technique whereby only rows or fields that are changed are fetched. This reduces load times of subsequent runs and improves efficiency of your pipelines.
+
+Assuming the same setup from [Simple Load](#example-simple-load), we can run incremental load with:
+```sh
+ingestr ingest \
+    --source-uri "dynamodb://dynamodb.ap-south-1.amazonaws.com?access_key_id=user&secret_access_key=pass" \
+    --source-table "absolute-armadillo" \
+    --dest-uri "duckdb://./animal.db" \
+    --dest-table "public.armadillo" \
+    --incremental-strategy "replace" \
+    --incremental-key "updated_at"
+```
+
+Assuming that `absolute-armadillo` table has a datetime field called `updated_at`, whenever you run this command, only rows with value greater than `MAX(updated_at)` from previous load will be fetched from DynamoDB.
+
+> [!WARNING]
+> DynamoDB doesn't support indexed range scans.
+> Whenever you run `ingestr ingest`, the whole table is scanned.
+> Although `ingestr` does specify a filter critiera, DynamoDB only applies
+> this _after_ running the table scan.

{ingestr-0.10.3 → ingestr-0.12.1}/docs/supported-sources/gorgias.md

@@ -23,7 +23,7 @@ The URI is used to connect to the Gorgias API for extracting data.
 ingestr ingest --source-table 'tickets' --source-uri $GORGIAS_URI --dest-uri $BIGQUERY_URI --interval-start 2024-06-19  --dest-table 'gorgias.ticket_messages' --loader-file-format jsonl
 
 # get all the customers and write them to `gorgias.customers` table on DuckDB
-ingestr ingest --source-table 'customers' --source-uri $GORGIAS_URI --dest-uri duckdb:///gorgias.duckdb --interval-start 2024-01-01  --dest-table 'gorgias.customers'
+ingestr ingest --source-table 'customers' --source-uri $GORGIAS_URI --dest-uri duckdb:///gorgias.duckdb --interval-start 2024-01-01  --dest-table 'dest.customers'
 ```
 
 ## Supported entities

{ingestr-0.10.3 → ingestr-0.12.1}/docs/supported-sources/gsheets.md

@@ -37,7 +37,7 @@ Once you complete the guide, you should have a service account JSON file and the
 Based on this assumption, here's a sample command that will copy the data from the Google Sheets spreadsheet into a DuckDB database:
 
 ```sh
-ingestr ingest --source-uri 'gsheets://?credentials_path=/path/to/file.json' --source-table 'fkdUQ2bjdNfUq2CA.Sheet1' --dest-uri duckdb:///gsheets.duckdb --dest-table 'gsheets.output'
+ingestr ingest --source-uri 'gsheets://?credentials_path=/path/to/file.json' --source-table 'fkdUQ2bjdNfUq2CA.Sheet1' --dest-uri duckdb:///gsheets.duckdb --dest-table 'dest.output'
 ```
 
 The result of this command will be a table in the `gsheets.duckdb` database.

{ingestr-0.10.3 → ingestr-0.12.1}/docs/supported-sources/kafka.md

@@ -32,7 +32,7 @@ ingestr ingest \
     --source-uri 'kafka://?bootstrap_servers=localhost:9092&group_id=test_group' \
     --source-table 'my-topic' \
     --dest-uri duckdb:///kafka.duckdb \
-    --dest-table 'kafka.my_topic'
+    --dest-table 'dest.my_topic'
 ```
 
 The result of this command will be a table in the `kafka.duckdb` database with JSON columns.

{ingestr-0.10.3 → ingestr-0.12.1}/docs/supported-sources/klaviyo.md

@@ -19,10 +19,10 @@ URI parameters:
 The URI is used to connect to the Klaviyo API for extracting data.
 
 ```bash
-ingestr ingest --source-table 'events' --source-uri 'klaviyo://?api_key=pk_test' --dest-uri duckdb:///klaviyo.duckdb --interval-start 2022-01-01 --dest-table 'klaviyo.events' --extract-parallelism 20
+ingestr ingest --source-table 'events' --source-uri 'klaviyo://?api_key=pk_test' --dest-uri duckdb:///klaviyo.duckdb --interval-start 2022-01-01 --dest-table 'dest.events' --extract-parallelism 20
 ```
 
-This command fetches all the events that are created/updated since 2022-01-01 and writes them to `klaviyo.events` table on DuckDB, using 20 parallel threads to improve performance and efficiently handle large data .
+This command fetches all the events that are created/updated since 2022-01-01 and writes them to `dest.events` table on DuckDB, using 20 parallel threads to improve performance and efficiently handle large data .
 
 ## Tables
 

{ingestr-0.10.3 → ingestr-0.12.1}/docs/supported-sources/notion.md

@@ -22,7 +22,7 @@ Notion requires a few steps to set up an integration, please follow the guide dl
 Once you complete the guide, you should have an API key, and the table ID to connect to. Let's say your API token is `secret_12345` and the database you'd like to connect to is `bfeaafc0c25f40a9asdasd672a9456f3`, here's a sample command that will copy the data from the Notion table into a DuckDB database:
 
 ```sh
-ingestr ingest --source-uri 'notion://?api_key=secret_12345' --source-table 'bfeaafc0c25f40a9asdasd672a9456f3' --dest-uri duckdb:///notion.duckdb --dest-table 'notion.output'
+ingestr ingest --source-uri 'notion://?api_key=secret_12345' --source-table 'bfeaafc0c25f40a9asdasd672a9456f3' --dest-uri duckdb:///notion.duckdb --dest-table 'dest.output'
 ```
 
 The result of this command will be a table in the `notion.duckdb` database with JSON columns. 

{ingestr-0.10.3 → ingestr-0.12.1}/docs/supported-sources/shopify.md

@@ -23,7 +23,7 @@ Shopify requires a few steps to set up an integration, please follow the guide d
 Once you complete the guide, you should have an API key and the store name to connect to. Let's say your API key is `shpkey_12345` and the store you'd like to connect to is `my-store`, here's a sample command that will copy the data from the Shopify store into a DuckDB database:
 
 ```sh
-ingestr ingest --source-uri 'shopify://my-store.myshopify.com?api_key=shpkey_12345' --source-table 'orders' --dest-uri duckdb:///shopify.duckdb --dest-table 'shopify.orders'
+ingestr ingest --source-uri 'shopify://my-store.myshopify.com?api_key=shpkey_12345' --source-table 'orders' --dest-uri duckdb:///shopify.duckdb --dest-table 'dest.orders'
 ```
 
 The result of this command will be a table in the `shopify.duckdb` database with JSON columns.

{ingestr-0.10.3 → ingestr-0.12.1}/docs/supported-sources/stripe.md

@@ -25,7 +25,7 @@ Stripe requires a few steps to set up an integration, please follow the guide dl
 Once you complete the guide, you should have an API key. Let's say your API key is `sk_test_12345`, here's a sample command that will copy the data from Stripe into a DuckDB database:
 
 ```sh
-ingestr ingest --source-uri 'stripe://?api_key=sk_test_12345' --source-table 'charges' --dest-uri duckdb:///stripe.duckdb --dest-table 'stripe.charges'
+ingestr ingest --source-uri 'stripe://?api_key=sk_test_12345' --source-table 'charges' --dest-uri duckdb:///stripe.duckdb --dest-table 'dest.charges'
 ```
 
 The result of this command will be a table in the `stripe.duckdb` database with JSON columns.

{ingestr-0.10.3 → ingestr-0.12.1}/docs/supported-sources/zendesk.md

@@ -39,7 +39,7 @@ Once you complete the guide, if you decide to use an OAuth token, you should hav
 ingestr ingest --source-uri "zendesk://:qVsbdiasVt@mycompany" \
 --source-table 'tickets' \
 --dest-uri 'duckdb:///zendesk.duckdb' \
---dest-table 'zendesk.tickets' \
+--dest-table 'dest.tickets' \
 --interval-start '2024-01-01'
 ```
 
@@ -49,7 +49,7 @@ If you decide to use an API Token, you should have a subdomain, email, and API t
 ingestr ingest --source-uri "zendesk://john@get.com:nbs123@mycompany" \
 --source-table 'tickets' \
 --dest-uri 'duckdb:///zendesk.duckdb' \
---dest-table 'zendesk.tickets' \
+--dest-table 'dest.tickets' \
 --interval-start '2024-01-01'
 ```
 

{ingestr-0.10.3 → ingestr-0.12.1}/ingestr/main.py

@@ -288,7 +288,6 @@ def ingest(
         ),
     ] = [],  # type: ignore
 ):
-
     # TODO(turtledev): can't we move this to the top of this file?
     import hashlib
     import tempfile
@@ -386,7 +385,7 @@ def ingest(
 
         factory = SourceDestinationFactory(source_uri, dest_uri)
         track(
-            "command_running", 
+            "command_running",
             {
                 "command": "ingest",
                 "source_type": factory.source_scheme,

{ingestr-0.10.3 → ingestr-0.12.1}/ingestr/src/adjust/__init__.py

@@ -67,13 +67,16 @@ def adjust_source(
             filters=filters,
         )
 
+    if not dimensions:
+        return campaigns, creatives
+
     merge_key = merge_key
+    type_hints = {}
     for dimension in REQUIRED_CUSTOM_DIMENSIONS:
         if dimension in dimensions:
             merge_key = dimension
             break
 
-    type_hints = {}
     for dimension in dimensions:
         if dimension in KNOWN_TYPE_HINTS:
             type_hints[dimension] = KNOWN_TYPE_HINTS[dimension]