ingestr 0.13.2__tar.gz → 0.13.4__tar.gz

ingestr-0.13.4/.github/workflows/release.yml

@@ -0,0 +1,79 @@
+name: "Release ingestr on pip and ghcr.io"
+
+on:
+  push:
+    tags:
+      - "v*.*.*"
+  
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  tests:
+    uses: ./.github/workflows/tests.yml
+  pip-release:
+    runs-on: ubuntu-latest
+    needs: tests
+    env:
+      TWINE_PASSWORD: ${{ secrets.PYPI_INGESTR_TOKEN }}
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+        
+        # github actions invoke a fresh shell
+        # for each step. This causes us to lose our
+        # venv context. So we run all our build commands
+        # in a single step to simplify our pipeline.
+      - name: build and release
+        run: |
+          pip install uv
+          uv venv
+          source .venv/bin/activate
+          uv pip install -r requirements-dev.txt
+          make build
+          make upload-release
+
+  docker-release:
+    needs: tests
+    if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64,linux/arm64/v8
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+    
+

{ingestr-0.13.2 → ingestr-0.13.4}/.github/workflows/tests.yml

@@ -2,9 +2,8 @@ on:
   push:
     branches:
       - main
-    tags:
-      - "*"
   workflow_dispatch: {}
+  workflow_call: {}
   pull_request:
     branches:
       - main
@@ -13,11 +12,6 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
-env:
-  REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
-
-
 jobs:
   tests:
     strategy:
@@ -70,39 +64,3 @@ jobs:
         run: make test-ci
       - name: check the formatting
         run: make lint-ci
-
-  build-and-push-image:
-    needs: tests
-    if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - name: Log in to the Container registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build and push Docker image
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          platforms: linux/amd64,linux/arm64,linux/arm64/v8
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max

{ingestr-0.13.2 → ingestr-0.13.4}/.gitignore

@@ -20,3 +20,4 @@ node_modules
 *.db
 *.db.wal
 *.sql
+ingestr/src/buildinfo.py

{ingestr-0.13.2 → ingestr-0.13.4}/Makefile

@@ -1,6 +1,9 @@
+SHELL := /bin/bash
 .ONESHELL:
 .PHONY: test lint format test-ci lint-ci build upload-release setup docker-shell
 
+BUILDINFO=ingestr/src/buildinfo.py
+
 venv: venv/touchfile
 
 venv/touchfile: requirements-dev.txt requirements.txt
@@ -36,7 +39,9 @@ lint-docs:
 tl: test lint
 
 build:
+	cat > ${BUILDINFO} <<< "version = \"$$(git describe --tags --abbrev=0)\""
 	rm -rf dist && python3 -m build
+	rm -f ${BUILDINFO}
 
 upload-release:
 	twine upload --verbose dist/*

{ingestr-0.13.2 → ingestr-0.13.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.2
+Version: 0.13.4
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -132,7 +132,6 @@ Pull requests are welcome. However, please open an issue first to discuss what y
 > After cloning `ingestr` make sure to run `make setup` to install githooks.
 
 ## Supported sources & destinations
-
 <table>
     <tr>
         <th></th>
@@ -142,33 +141,28 @@ Pull requests are welcome. However, please open an issue first to discuss what y
     <tr>
         <td colspan="3" style='text-align:center;'><strong>Databases</strong></td>
     </tr>
-    <tr>
-        <td>Postgres</td>
-        <td>✅</td>
-        <td>✅</td>
-    </tr>
     <tr>
         <td>BigQuery</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
     <tr>
-        <td>Snowflake</td>
-        <td>✅</td>
+        <td>ClickHouse</td>
+        <td>❌</td>
         <td>✅</td>
     </tr>
     <tr>
-        <td>Redshift</td>
+        <td>Databricks</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
     <tr>
-        <td>Databricks</td>
+        <td>DuckDB</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
     <tr>
-        <td>DuckDB</td>
+        <td>Local CSV file</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
@@ -178,12 +172,12 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
     </tr>
     <tr>
-        <td>Local CSV file</td>
-        <td>✅</td>
+        <td>MongoDB</td>
         <td>✅</td>
+        <td>❌</td>
     </tr>
     <tr>
-        <td>MongoDB</td>
+        <td>MySQL</td>
         <td>✅</td>
         <td>❌</td>
     </tr>
@@ -192,18 +186,28 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>❌</td>
     </tr>
+    <tr>
+        <td>Postgres</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+     <tr>
+        <td>Redshift</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
     <tr>
         <td>SAP Hana</td>
         <td>✅</td>
         <td>❌</td>
     </tr>
     <tr>
-        <td>SQLite</td>
+        <td>Snowflake</td>
+        <td>✅</td>
         <td>✅</td>
-        <td>❌</td>
     </tr>
     <tr>
-        <td>MySQL</td>
+        <td>SQLite</td>
         <td>✅</td>
         <td>❌</td>
     </tr>
@@ -218,11 +222,21 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Apache Kafka</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>AppsFlyer</td>
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>App Store</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Asana</td>
         <td>✅</td>
@@ -243,6 +257,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Github</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Gorgias</td>
         <td>✅</td>
@@ -254,7 +273,17 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>-</td>
     </tr>
     <tr>
-        <td>HubSpot</td>
+        <td>Google Ads</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Google Analytics</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Intercom</td>
         <td>✅</td>
         <td>-</td>
     </tr>
@@ -263,6 +292,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>LinkedIn Ads</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Notion</td>
         <td>✅</td>
@@ -288,6 +322,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>TikTok Ads</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Zendesk</td>
         <td>✅</td>

{ingestr-0.13.2 → ingestr-0.13.4}/README.md

@@ -68,7 +68,6 @@ Pull requests are welcome. However, please open an issue first to discuss what y
 > After cloning `ingestr` make sure to run `make setup` to install githooks.
 
 ## Supported sources & destinations
-
 <table>
     <tr>
         <th></th>
@@ -78,33 +77,28 @@ Pull requests are welcome. However, please open an issue first to discuss what y
     <tr>
         <td colspan="3" style='text-align:center;'><strong>Databases</strong></td>
     </tr>
-    <tr>
-        <td>Postgres</td>
-        <td>✅</td>
-        <td>✅</td>
-    </tr>
     <tr>
         <td>BigQuery</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
     <tr>
-        <td>Snowflake</td>
-        <td>✅</td>
+        <td>ClickHouse</td>
+        <td>❌</td>
         <td>✅</td>
     </tr>
     <tr>
-        <td>Redshift</td>
+        <td>Databricks</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
     <tr>
-        <td>Databricks</td>
+        <td>DuckDB</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
     <tr>
-        <td>DuckDB</td>
+        <td>Local CSV file</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
@@ -114,12 +108,12 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
     </tr>
     <tr>
-        <td>Local CSV file</td>
-        <td>✅</td>
+        <td>MongoDB</td>
         <td>✅</td>
+        <td>❌</td>
     </tr>
     <tr>
-        <td>MongoDB</td>
+        <td>MySQL</td>
         <td>✅</td>
         <td>❌</td>
     </tr>
@@ -128,18 +122,28 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>❌</td>
     </tr>
+    <tr>
+        <td>Postgres</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+     <tr>
+        <td>Redshift</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
     <tr>
         <td>SAP Hana</td>
         <td>✅</td>
         <td>❌</td>
     </tr>
     <tr>
-        <td>SQLite</td>
+        <td>Snowflake</td>
+        <td>✅</td>
         <td>✅</td>
-        <td>❌</td>
     </tr>
     <tr>
-        <td>MySQL</td>
+        <td>SQLite</td>
         <td>✅</td>
         <td>❌</td>
     </tr>
@@ -154,11 +158,21 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Apache Kafka</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>AppsFlyer</td>
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>App Store</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Asana</td>
         <td>✅</td>
@@ -179,6 +193,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Github</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Gorgias</td>
         <td>✅</td>
@@ -190,7 +209,17 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>-</td>
     </tr>
     <tr>
-        <td>HubSpot</td>
+        <td>Google Ads</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Google Analytics</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Intercom</td>
         <td>✅</td>
         <td>-</td>
     </tr>
@@ -199,6 +228,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>LinkedIn Ads</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Notion</td>
         <td>✅</td>
@@ -224,6 +258,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>TikTok Ads</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Zendesk</td>
         <td>✅</td>

{ingestr-0.13.2 → ingestr-0.13.4}/docs/.vitepress/config.mjs

@@ -96,6 +96,7 @@ export default defineConfig({
               { text: "Airtable", link: "/supported-sources/airtable.md" },
               { text: "AppsFlyer", link: "/supported-sources/appsflyer.md" },
               { text: "Apple App Store", link: "/supported-sources/appstore.md"},
+              { text: "Applovin", link: "/supported-sources/applovin.md"},
               { text: "Asana", link: "/supported-sources/asana.md" },
               { text: "Chess.com", link: "/supported-sources/chess.md" },
               { text: "DynamoDB", link: "/supported-sources/dynamodb.md" },

ingestr-0.13.4/docs/supported-sources/applovin.md

@@ -0,0 +1,144 @@
+# Applovin
+[AppLovin](https://www.applovin.com/) Corporation is an American mobile technology company headquartered in Palo Alto, California. AppLovin enables developers of all sizes to market, monetize, analyze and publish their apps through its mobile advertising, marketing, and analytics platforms MAX, AppDiscovery, and SparkLabs.
+
+`ingestr` allows ingesting data from [AppDiscovery reporting API](https://developers.applovin.com/en/app-discovery/api/reporting-api/)
+
+## URI Format
+
+The URI format for Applovin is as follows:
+```
+applovin://?api_key=<your_api_key>
+```
+
+URI Parameters:
+* `api_key`. report key generated from your [applovin account](https://www.applovin.com/analytics#keys).
+
+## Setting up Applovin Integration
+
+### Generate a Report Key
+You can generate a report key from your [analytics dashboard](https://www.applovin.com/analytics#keys).
+
+### Example: Loading Publisher Report
+
+For this example, we'll assume that:
+* `api_key` is `api_key_0`
+
+We will run `ingestr` to save this data to a [duckdb](https://duckdb.org/) database called `report.db` under the table name `public.publisher_report`.
+
+```sh
+ingestr ingest \
+    --source-uri "applovin://?api_key=api_key_0
+    --source-table "publisher-report" \
+    --dest-uri "duckdb:///report.db"  \
+    --dest-table "public.publisher_report" 
+```
+
+### Example: Incremental loading
+
+We will extend the [Loading Publisher Report](#example-loading-publisher-report) example to demonstrate incremental loading.
+
+First, we run the example with a start date of `2025-01-01` and an end date of `2025-01-05`
+
+```sh
+ingestr ingest \
+    --source-uri "applovin://?api_key=api_key_0
+    --source-table "publisher-report" \
+    --dest-uri "duckdb:///report.db"  \
+    --dest-table "public.publisher_report" \
+    --interval-start "2025-01-01" \
+    --interval-end "2025-01-05" 
+```
+
+We can query the database to see which dates the data was ingested for:
+
+```sh
+$ duckdb report.db 'select day from public.publisher_report group by 1'
+┌────────────┐
+│    day     │
+│    date    │
+├────────────┤
+│ 2025-01-01 │
+│ 2025-01-02 │
+│ 2025-01-03 │
+│ 2025-01-04 │
+│ 2025-01-05 │
+└────────────┘
+```
+Now, we will run `ingestr` again, but we will omit start and end date to demonstrate an incremental load.
+
+```sh
+ingestr ingest \
+    --source-uri "applovin://?api_key=api_key_0
+    --source-table "publisher-report" \
+    --dest-uri "duckdb:///report.db"  \
+    --dest-table "public.publisher_report" 
+```
+
+Now we can check the database again, and we will see that the data was loaded for the rest of the days automatically.
+```sh
+$ duckdb report.db 'select day from public.publisher_report group by 1'
+┌────────────┐
+│    day     │
+│    date    │
+├────────────┤
+│ 2025-01-01 │
+│ 2025-01-02 │
+│ 2025-01-03 │
+│ 2025-01-04 │
+│ 2025-01-05 │
+│ 2025-01-06 │
+│     .      │
+│     .      │
+│     .      │
+│ 2025-01-28 │
+│ 2025-01-29 │
+│ 2025-01-30 │
+│ 2025-01-31 │
+├────────────┤
+│  35 rows   │
+└────────────┘
+```
+
+## Tables
+
+| Name | Description |
+| --- | --- |
+| `publisher-report` | Provides daily metrics from the `report` end point using the report_type `publisher` |
+| `advertiser-report` | Provides daily metrics from the `report` end point using the report_type `advertiser`|
+| `advertiser-probabilistic-report` | Provides daily metrics from the `probabilisticReport` end point using the report_type `advertiser` |
+| `advertiser-ska-report` | Provides daily metrics from the `skaReport` end point using the report_type `advertiser` |
+
+## Custom Reports
+
+`applovin` source supports custom reports. You can pass a custom report definition to `--source-table` and it will dynamically create a report for you.
+
+The format of a custom report looks like the following:
+```
+custom:{endpoint}:{report_type}:{columns}
+```
+Where:
+* `{endpoint}` is the API endpoint for applovin reports (one of `report`, `probabilisticReport` or `skaReport`)
+* `{report_type}` is the [report type](https://developers.applovin.com/en/app-discovery/api/reporting-api#required-parameters) (one of `publisher` or `advertiser`)
+* `{columns}` are the [columns](https://developers.applovin.com/en/app-discovery/api/reporting-api#allowed-publisher-columns) of the given report type.
+
+
+### Custom Report Example
+For this example, we will ingest data from `report` end point with the report type `publisher`.
+
+We want to obtain the following columns:
+* ad_type
+* clicks
+* country
+
+To achieve this, we can pass the custom report defintion in `--source-table`
+
+```sh
+ingestr ingest \
+    --source-uri "applovin://?api_key=api_key_0
+    --source-table "custom:report:publisher:ad_type,clicks,country" \
+    --dest-uri "duckdb:///report.db"  \
+    --dest-table "public.custom_report" 
+```
+
+> [!NOTE]
+> The `day` column is automatically added to any custom report if it is not specified in the custom report definition.