ingestr 0.13.29__tar.gz → 0.13.30__tar.gz

{ingestr-0.13.29 → ingestr-0.13.30}/.gitignore

@@ -20,4 +20,5 @@ node_modules
 *.db
 *.db.wal
 *.sql
-ingest.sh
+ingest.sh
+test.env

{ingestr-0.13.29 → ingestr-0.13.30}/Makefile

@@ -22,13 +22,13 @@ deps-ci:
 	uv pip install --system -r requirements-dev.txt
 
 test-ci:
-	TESTCONTAINERS_RYUK_DISABLED=true pytest -n auto -x -rP -vv --tb=short --durations=10 --cov=ingestr --no-cov-on-fail
+	set -a; source test.env; set +a; TESTCONTAINERS_RYUK_DISABLED=true pytest -n auto -x -rP -vv --tb=short --durations=10 --cov=ingestr --no-cov-on-fail
 
 test : venv lock-deps
 	. venv/bin/activate; $(MAKE) test-ci
 
 test-specific: venv lock-deps
-	. venv/bin/activate; TESTCONTAINERS_RYUK_DISABLED=true pytest -n auto  -rP -vv --tb=short --capture=no -k $(test)
+	. venv/bin/activate; set -a; source test.env; set +a; TESTCONTAINERS_RYUK_DISABLED=true pytest -n auto  -rP -vv --tb=short --capture=no -k $(test)
 
 lint-ci:
 	ruff format ingestr && ruff check ingestr --fix

{ingestr-0.13.29 → ingestr-0.13.30}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.29
+Version: 0.13.30
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.29 → ingestr-0.13.30}/docs/.vitepress/config.mjs

@@ -29,6 +29,9 @@ export default defineConfig({
       { text: "Getting started", link: "/getting-started/quickstart.md" },
     ],
     outline: "deep",
+    search: {
+      provider: 'local'
+    },
 
     sidebar: [
       {

ingestr-0.13.30/docs/supported-sources/appsflyer.md

@@ -0,0 +1,58 @@
+# AppsFlyer
+
+[AppsFlyer](https://www.appsflyer.com/) is a mobile marketing analytics and attribution platform that helps businesses track, measure, and optimize their app marketing efforts across various channels.
+
+ingestr supports AppsFlyer as a source.
+
+> [!WARNING]
+> AppsFlyer uses different names for input dimensions vs their name in the output schema. For instance, in order to obtain campaign information, you need to use the `c` dimension; however, in the output schema, the resulting column will be called `campaign`.
+
+
+## URI Format
+
+The URI format for AppsFlyer is as follows:
+
+```plaintext
+appsflyer://?api_key=<api-key>
+```
+
+An API token is required to retrieve reports from the AppsFlyer API, please [follow the guide to obtain an API key](https://support.appsflyer.com/hc/en-us/articles/360004562377-Managing-AppsFlyer-tokens)
+
+Let's say your API key is `ey123`, here's a sample command that will copy the data from AppsFlyer into a DuckDB database:
+
+```bash
+ingestr ingest 
+    --source-uri 'appsflyer://?api_key=ey123' 
+    --source-table 'campaigns' 
+    --dest-uri duckdb:///appsflyer.duckdb 
+    --dest-table 'appsflyer.output'
+```
+
+The result of this command will be a table in the `appsflyer.duckdb` database.
+
+## Supported Tables
+
+ingestr integrates with the [Master Report API](https://dev.appsflyer.com/hc/reference/master_api_get) of AppsFlyer, which allows you to retrieve data for the following tables:
+
+- `campaigns`: Retrieves data for campaigns, detailing the app's costs, loyal users, total installs, and revenue over multiple days.
+- `creatives`: Retrieves data for a creative asset, including revenue and cost.
+- `custom:<dimensions>:<metrics>`: Retrieves data for custom tables, which can be specified by the user.
+
+Use these as `--source-table` parameter in the `ingestr ingest` command.
+
+### Custom Tables
+
+You can also ingest custom tables by providing a list of dimensions and metrics.
+
+The table format is as follows:
+
+```plaintext
+custom:<dimension1>,<dimension2>,<metric1>,<metric2>
+```
+
+This will automatically generate a table with the dimensions and metrics you provided.
+
+For custom tables, ingestr will use the given dimensions as the primary key to deduplicate the data.
+
+> [!NOTE]
+> ingestr will add `install_time` as the primary key to the table by default if it is not provided as one of the dimensions.

ingestr-0.13.30/ingestr/src/appsflyer/__init__.py

@@ -0,0 +1,325 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from ingestr.src.appsflyer.client import AppsflyerClient
+
+DIMENSION_RESPONSE_MAPPING = {
+    "c": "campaign",
+    "af_adset_id": "adset_id",
+    "af_adset": "adset",
+    "af_ad_id": "ad_id",
+}
+HINTS = {
+    "app_id": {
+        "data_type": "text",
+        "nullable": False,
+    },
+    "campaign": {
+        "data_type": "text",
+        "nullable": False,
+    },
+    "geo": {
+        "data_type": "text",
+        "nullable": False,
+    },
+    "cost": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": False,
+    },
+    "clicks": {
+        "data_type": "bigint",
+        "nullable": False,
+    },
+    "impressions": {
+        "data_type": "bigint",
+        "nullable": False,
+    },
+    "average_ecpi": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": False,
+    },
+    "installs": {
+        "data_type": "bigint",
+        "nullable": False,
+    },
+    "retention_day_7": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": False,
+    },
+    "retention_day_14": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": False,
+    },
+    "cohort_day_1_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_1_total_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_3_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_3_total_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_7_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_7_total_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_14_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_14_total_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_21_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_21_total_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "install_time": {
+        "data_type": "date",
+        "nullable": False,
+    },
+    "loyal_users": {
+        "data_type": "bigint",
+        "nullable": False,
+    },
+    "revenue": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "roi": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "uninstalls": {
+        "data_type": "bigint",
+        "nullable": True,
+    },
+}
+
+CAMPAIGNS_DIMENSIONS = ["c", "geo", "app_id", "install_time"]
+CAMPAIGNS_METRICS = [
+    "average_ecpi",
+    "clicks",
+    "cohort_day_1_revenue_per_user",
+    "cohort_day_1_total_revenue_per_user",
+    "cohort_day_14_revenue_per_user",
+    "cohort_day_14_total_revenue_per_user",
+    "cohort_day_21_revenue_per_user",
+    "cohort_day_21_total_revenue_per_user",
+    "cohort_day_3_revenue_per_user",
+    "cohort_day_3_total_revenue_per_user",
+    "cohort_day_7_revenue_per_user",
+    "cohort_day_7_total_revenue_per_user",
+    "cost",
+    "impressions",
+    "installs",
+    "loyal_users",
+    "retention_day_7",
+    "revenue",
+    "roi",
+    "uninstalls",
+]
+
+CREATIVES_DIMENSIONS = [
+    "c",
+    "geo",
+    "app_id",
+    "install_time",
+    "af_adset_id",
+    "af_adset",
+    "af_ad_id",
+]
+CREATIVES_METRICS = [
+    "impressions",
+    "clicks",
+    "installs",
+    "cost",
+    "revenue",
+    "average_ecpi",
+    "loyal_users",
+    "uninstalls",
+    "roi",
+]
+
+
+@dlt.source(max_table_nesting=0)
+def appsflyer_source(
+    api_key: str,
+    start_date: str,
+    end_date: str,
+    dimensions: list[str],
+    metrics: list[str],
+) -> Iterable[DltResource]:
+    client = AppsflyerClient(api_key)
+
+    @dlt.resource(
+        write_disposition="merge",
+        merge_key="install_time",
+        columns=make_hints(CAMPAIGNS_DIMENSIONS, CAMPAIGNS_METRICS),
+    )
+    def campaigns(
+        datetime=dlt.sources.incremental(
+            "install_time",
+            initial_value=(
+                start_date
+                if start_date
+                else pendulum.today().subtract(days=30).format("YYYY-MM-DD")
+            ),
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        end = (
+            datetime.end_value
+            if datetime.end_value
+            else pendulum.now().format("YYYY-MM-DD")
+        )
+
+        yield from client._fetch_data(
+            from_date=datetime.last_value,
+            to_date=end,
+            dimensions=CAMPAIGNS_DIMENSIONS,
+            metrics=CAMPAIGNS_METRICS,
+        )
+
+    @dlt.resource(
+        write_disposition="merge",
+        merge_key="install_time",
+        columns=make_hints(CREATIVES_DIMENSIONS, CREATIVES_METRICS),
+    )
+    def creatives(
+        datetime=dlt.sources.incremental(
+            "install_time",
+            initial_value=(
+                start_date
+                if start_date
+                else pendulum.today().subtract(days=30).format("YYYY-MM-DD")
+            ),
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        end = (
+            datetime.end_value
+            if datetime.end_value
+            else pendulum.now().format("YYYY-MM-DD")
+        )
+        yield from client._fetch_data(
+            datetime.last_value,
+            end,
+            dimensions=CREATIVES_DIMENSIONS,
+            metrics=CREATIVES_METRICS,
+        )
+
+    primary_keys = []
+    if "install_time" not in dimensions:
+        dimensions.append("install_time")
+        primary_keys.append("install_time")
+
+    for dimension in dimensions:
+        if dimension in DIMENSION_RESPONSE_MAPPING:
+            primary_keys.append(DIMENSION_RESPONSE_MAPPING[dimension])
+        else:
+            primary_keys.append(dimension)
+
+    @dlt.resource(
+        write_disposition="merge",
+        primary_key=primary_keys,
+        columns=make_hints(dimensions, metrics),
+    )
+    def custom(
+        datetime=dlt.sources.incremental(
+            "install_time",
+            initial_value=(
+                start_date
+                if start_date
+                else pendulum.today().subtract(days=30).format("YYYY-MM-DD")
+            ),
+            end_value=end_date,
+        ),
+    ):
+        end = (
+            datetime.end_value
+            if datetime.end_value
+            else pendulum.now().format("YYYY-MM-DD")
+        )
+        res = client._fetch_data(
+            from_date=datetime.last_value,
+            to_date=end,
+            dimensions=dimensions,
+            metrics=metrics,
+        )
+        yield from res
+
+    return campaigns, creatives, custom
+
+
+def make_hints(dimensions: list[str], metrics: list[str]):
+    campaign_hints = {}
+    for dimension in dimensions:
+        resp_key = dimension
+        if dimension in DIMENSION_RESPONSE_MAPPING:
+            resp_key = DIMENSION_RESPONSE_MAPPING[dimension]
+
+        if resp_key in HINTS:
+            campaign_hints[resp_key] = HINTS[resp_key]
+
+    for metric in metrics:
+        if metric in HINTS:
+            campaign_hints[metric] = HINTS[metric]
+
+    return campaign_hints

ingestr-0.13.30/ingestr/src/appsflyer/client.py

@@ -0,0 +1,111 @@
+from typing import Optional
+
+import requests
+from dlt.sources.helpers.requests import Client
+from requests.exceptions import HTTPError
+
+
+class AppsflyerClient:
+    def __init__(self, api_key: str):
+        self.api_key = api_key
+        self.uri = "https://hq1.appsflyer.com/api/master-agg-data/v4/app/all"
+
+    def __get_headers(self):
+        return {
+            "Authorization": f"{self.api_key}",
+            "accept": "text/json",
+        }
+
+    def _fetch_data(
+        self,
+        from_date: str,
+        to_date: str,
+        dimensions: list[str],
+        metrics: list[str],
+        maximum_rows=1000000,
+    ):
+        excluded_metrics = exclude_metrics_for_date_range(metrics, from_date, to_date)
+        included_metrics = [
+            metric for metric in metrics if metric not in excluded_metrics
+        ]
+
+        params = {
+            "from": from_date,
+            "to": to_date,
+            "groupings": ",".join(dimensions),
+            "kpis": ",".join(included_metrics),
+            "format": "json",
+            "maximum_rows": maximum_rows,
+        }
+
+        def retry_on_limit(
+            response: Optional[requests.Response], exception: Optional[BaseException]
+        ) -> bool:
+            return (
+                isinstance(response, requests.Response) and response.status_code == 429
+            )
+
+        request_client = Client(
+            request_timeout=10.0,
+            raise_for_status=False,
+            retry_condition=retry_on_limit,
+            request_max_attempts=12,
+            request_backoff_factor=2,
+        ).session
+
+        try:
+            response = request_client.get(
+                url=self.uri, headers=self.__get_headers(), params=params
+            )
+
+            if response.status_code == 200:
+                result = response.json()
+                yield standardize_keys(result, excluded_metrics)
+            else:
+                raise HTTPError(
+                    f"Request failed with status code: {response.status_code}: {response.text}"
+                )
+
+        except requests.RequestException as e:
+            raise HTTPError(f"Request failed: {e}")
+
+
+def standardize_keys(data: list[dict], excluded_metrics: list[str]) -> list[dict]:
+    def fix_key(key: str) -> str:
+        return key.lower().replace("-", "").replace("  ", "_").replace(" ", "_")
+
+    standardized = []
+    for item in data:
+        standardized_item = {}
+        for key, value in item.items():
+            standardized_item[fix_key(key)] = value
+
+        for metric in excluded_metrics:
+            if metric not in standardized_item:
+                standardized_item[fix_key(metric)] = None
+
+        standardized.append(standardized_item)
+
+    return standardized
+
+
+def exclude_metrics_for_date_range(
+    metrics: list[str], from_date: str, to_date: str
+) -> list[str]:
+    """
+    Some of the cohort metrics are not available if there hasn't been enough time to have data for that cohort.
+    This means if you request data for yesterday with cohort day 7 metrics, you will get an error because 7 days hasn't passed yet.
+    One would expect the API to handle this gracefully, but it doesn't.
+
+    This function will exclude the metrics that are not available for the given date range.
+    """
+    import pendulum
+
+    excluded_metrics = []
+    days_between_today_and_end = (pendulum.now() - pendulum.parse(to_date)).days  # type: ignore
+    for metric in metrics:
+        if "cohort_day_" in metric:
+            day_count = int(metric.split("_")[2])
+            if days_between_today_and_end <= day_count:
+                excluded_metrics.append(metric)
+    return excluded_metrics

ingestr-0.13.30/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.30"

{ingestr-0.13.29 → ingestr-0.13.30}/ingestr/src/sources.py

@@ -52,7 +52,6 @@ from ingestr.src.adjust.adjust_helpers import parse_filters
 from ingestr.src.airtable import airtable_source
 from ingestr.src.applovin import applovin_source
 from ingestr.src.applovin_max import applovin_max_source
-from ingestr.src.appsflyer._init_ import appsflyer_source
 from ingestr.src.appstore import app_store
 from ingestr.src.appstore.client import AppStoreConnectClient
 from ingestr.src.arrow import memory_mapped_arrow
@@ -816,7 +815,15 @@ class HubspotSource:
                 custom_object=endpoint,
             ).with_resources("custom")
 
-        elif table in ["contacts", "companies", "deals", "tickets", "products", "quotes", "schemas"]:
+        elif table in [
+            "contacts",
+            "companies",
+            "deals",
+            "tickets",
+            "products",
+            "quotes",
+            "schemas",
+        ]:
             endpoint = table
         else:
             raise ValueError(
@@ -1035,6 +1042,8 @@ class AppsflyerSource:
         return True
 
     def dlt_source(self, uri: str, table: str, **kwargs):
+        from ingestr.src.appsflyer import appsflyer_source
+
         if kwargs.get("incremental_key"):
             raise ValueError(
                 "Appsflyer_Source takes care of incrementality on its own, you should not provide incremental_key"
@@ -1047,22 +1056,27 @@ class AppsflyerSource:
         if not api_key:
             raise ValueError("api_key in the URI is required to connect to Appsflyer")
 
-        resource = None
-        if table in ["campaigns", "creatives"]:
-            resource = table
-        else:
-            raise ValueError(
-                f"Resource '{table}' is not supported for Appsflyer source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
-            )
-
-        start_date = kwargs.get("interval_start") or "2024-01-02"
-        end_date = kwargs.get("interval_end") or "2024-01-29"
+        start_date = kwargs.get("interval_start")
+        end_date = kwargs.get("interval_end")
+        dimensions = []
+        metrics = []
+        if table.startswith("custom:"):
+            fields = table.split(":", 3)
+            if len(fields) != 3:
+                raise ValueError(
+                    "Invalid Adjust custom table format. Expected format: custom:<dimensions>:<metrics>"
+                )
+            dimensions = fields[1].split(",")
+            metrics = fields[2].split(",")
+            table = "custom"
 
         return appsflyer_source(
             api_key=api_key[0],
-            start_date=start_date,
-            end_date=end_date,
-        ).with_resources(resource)
+            start_date=start_date.strftime("%Y-%m-%d") if start_date else None,  # type: ignore
+            end_date=end_date.strftime("%Y-%m-%d") if end_date else None,  # type: ignore
+            dimensions=dimensions,
+            metrics=metrics,
+        ).with_resources(table)
 
 
 class ZendeskSource:
@@ -1414,16 +1428,16 @@ class GoogleAnalyticsSource:
         cred_base64 = source_fields.get("credentials_base64")
 
         if not cred_path and not cred_base64:
-            raise ValueError("credentials_path or credentials_base64 is required to connect Google Analytics")
+            raise ValueError(
+                "credentials_path or credentials_base64 is required to connect Google Analytics"
+            )
 
         credentials = {}
         if cred_path:
             with open(cred_path[0], "r") as f:
                 credentials = json.load(f)
         elif cred_base64:
-            credentials = json.loads(
-                base64.b64decode(cred_base64[0]).decode("utf-8")
-            )
+            credentials = json.loads(base64.b64decode(cred_base64[0]).decode("utf-8"))
 
         property_id = source_fields.get("property_id")
         if not property_id:

ingestr-0.13.30/test.env.template

@@ -0,0 +1,4 @@
+# This file is used to set environment variables for the tests.
+# If you create a test.env file based on this template then the integration tests will use those env variables automatically.
+
+export INGESTR_TEST_APPSFLYER_TOKEN="your_appsflyer_token_here"

ingestr-0.13.29/docs/supported-sources/appsflyer.md

@@ -1,33 +0,0 @@
-# AppsFlyer
-
-[AppsFlyer](https://www.appsflyer.com/) is a mobile marketing analytics and attribution platform that helps businesses track, measure, and optimize their app marketing efforts across various channels.
-
-ingestr supports AppsFlyer as a source.
-
-The URI format for AppsFlyer is as follows:
-
-```plaintext
-appsflyer://?api_key=<api-key>
-```
-
-An API token is required to retrieve reports from the AppsFlyer API. Please follow the guide to [obtain a API key](https://support.appsflyer.com/hc/en-us/articles/360004562377-Managing-AppsFlyer-tokens)
-
-Once you complete the guide, you should have an API key. Let's say your API key is `ey123`, here's a sample command that will copy the data from AppsFlyer into a DuckDB database:
-
-```bash
-ingestr ingest 
-    --source-uri 'appsflyer://?api_key=ey123' 
-    --source-table 'campaigns' 
-    --dest-uri duckdb:///appsflyer.duckdb 
-    --dest-table 'appsflyer.output'
-```
-
-The result of this command will be a table in the `appsflyer.duckdb` database.
-
-## Tables
-AppsFlyer source allows ingesting the following sources into separate tables:
-
-- `campaigns`: Retrieves data for campaigns, detailing the app's costs, loyal users, total installs, and revenue over multiple days.
-- `creatives`: Retrieves data for a creative asset, including revenue and cost.
-
-Use these as `--source-table` parameter in the `ingestr ingest` command.