ingestr 0.13.2__py3-none-any.whl → 0.14.104__py3-none-any.whl

ingestr/src/applovin/__init__.py

@@ -0,0 +1,262 @@
+from datetime import datetime, timedelta, timezone
+from enum import Enum
+from typing import Dict, List, Optional
+
+import dlt
+from dlt.sources.rest_api import EndpointResource, RESTAPIConfig, rest_api_resources
+from requests import Response
+
+
+class InvalidCustomReportError(Exception):
+    def __init__(self):
+        super().__init__(
+            "Custom report should be in the format 'custom:{endpoint}:{report_type}:{dimensions}"
+        )
+
+
+class ClientError(Exception):
+    pass
+
+
+TYPE_HINTS = {
+    "application_is_hidden": {"data_type": "bool"},
+    "average_cpa": {"data_type": "double"},
+    "average_cpc": {"data_type": "double"},
+    "campaign_bid_goal": {"data_type": "double"},
+    "campaign_roas_goal": {"data_type": "double"},
+    "clicks": {"data_type": "bigint"},
+    "conversions": {"data_type": "bigint"},
+    "conversion_rate": {"data_type": "double"},
+    "cost": {"data_type": "double"},  # assuming float.
+    "ctr": {"data_type": "double"},
+    "day": {"data_type": "date"},
+    "first_purchase": {"data_type": "bigint"},
+    "ecpm": {"data_type": "double"},
+    "impressions": {"data_type": "bigint"},
+    "installs": {"data_type": "bigint"},
+    "revenue": {"data_type": "double"},
+    "redownloads": {"data_type": "bigint"},
+    "sales": {"data_type": "double"},  # assuming float.
+}
+
+
+class ReportType(Enum):
+    PUBLISHER = "publisher"
+    ADVERTISER = "advertiser"
+
+
+REPORT_SCHEMA: Dict[ReportType, List[str]] = {
+    ReportType.PUBLISHER: [
+        "ad_type",
+        "application",
+        "application_is_hidden",
+        "bidding_integration",
+        "clicks",
+        "country",
+        "ctr",
+        "day",
+        "device_type",
+        "ecpm",
+        "impressions",
+        "package_name",
+        "placement_type",
+        "platform",
+        "revenue",
+        "size",
+        "store_id",
+        "zone",
+        "zone_id",
+    ],
+    ReportType.ADVERTISER: [
+        "ad",
+        "ad_creative_type",
+        "ad_id",
+        "ad_type",
+        "average_cpa",
+        "average_cpc",
+        "campaign",
+        "campaign_ad_type",
+        "campaign_bid_goal",
+        "campaign_id_external",
+        "campaign_package_name",
+        "campaign_roas_goal",
+        "campaign_store_id",
+        "campaign_type",
+        "clicks",
+        "conversions",
+        "conversion_rate",
+        "cost",
+        "country",
+        "creative_set",
+        "creative_set_id",
+        "ctr",
+        "custom_page_id",
+        "day",
+        "device_type",
+        "external_placement_id",
+        "first_purchase",
+        "impressions",
+        "installs",
+        "optimization_day_target",
+        "placement_type",
+        "platform",
+        "redownloads",
+        "sales",
+        "size",
+        "target_event",
+        "traffic_source",
+    ],
+}
+
+PROBABILISTIC_REPORT_EXCLUDE = [
+    "installs",
+    "redownloads",
+]
+
+
+@dlt.source
+def applovin_source(
+    api_key: str,
+    start_date: str,
+    end_date: Optional[str],
+    custom: Optional[str],
+):
+    backfill = False
+    if end_date is None:
+        backfill = True
+
+        # use the greatest of yesterday and start_date
+        end_date = max(
+            datetime.now(timezone.utc) - timedelta(days=1),
+            datetime.fromisoformat(start_date).replace(tzinfo=timezone.utc),
+        ).strftime("%Y-%m-%d")
+
+    config: RESTAPIConfig = {
+        "client": {
+            "base_url": "https://r.applovin.com/",
+            "auth": {
+                "type": "api_key",
+                "name": "api_key",
+                "location": "query",
+                "api_key": api_key,
+            },
+        },
+        "resource_defaults": {
+            "write_disposition": "merge",
+            "endpoint": {
+                "incremental": {
+                    "cursor_path": "day",
+                    "start_param": "start",
+                    "initial_value": start_date,
+                    "range_start": "closed",
+                    "range_end": "closed",
+                },
+                "params": {
+                    "format": "json",
+                    "end": end_date,
+                },
+                "paginator": "single_page",
+                "response_actions": [
+                    http_error_handler,
+                ],
+            },
+        },
+        "resources": [
+            resource(
+                "publisher-report",
+                "report",
+                REPORT_SCHEMA[ReportType.PUBLISHER],
+                ReportType.PUBLISHER,
+            ),
+            resource(
+                "advertiser-report",
+                "report",
+                REPORT_SCHEMA[ReportType.ADVERTISER],
+                ReportType.ADVERTISER,
+            ),
+            resource(
+                "advertiser-probabilistic-report",
+                "probabilisticReport",
+                exclude(
+                    REPORT_SCHEMA[ReportType.ADVERTISER], PROBABILISTIC_REPORT_EXCLUDE
+                ),
+                ReportType.ADVERTISER,
+            ),
+            resource(
+                "advertiser-ska-report",
+                "skaReport",
+                REPORT_SCHEMA[ReportType.ADVERTISER],
+                ReportType.ADVERTISER,
+            ),
+        ],
+    }
+
+    if custom:
+        custom_report = custom_report_from_spec(custom)
+        config["resources"].append(custom_report)
+
+    if backfill:
+        config["resource_defaults"]["endpoint"]["incremental"]["end_value"] = end_date  # type: ignore
+
+    yield from rest_api_resources(config)
+
+
+def resource(
+    name: str,
+    endpoint: str,
+    dimensions: List[str],
+    report_type: ReportType,
+) -> EndpointResource:
+    return {
+        "name": name,
+        "columns": build_type_hints(dimensions),
+        "merge_key": "day",
+        "endpoint": {
+            "path": endpoint,
+            "params": {
+                "report_type": report_type.value,
+                "columns": ",".join(dimensions),
+            },
+        },
+    }
+
+
+def custom_report_from_spec(spec: str) -> EndpointResource:
+    parts = spec.split(":")
+    if len(parts) != 4:
+        raise InvalidCustomReportError()
+
+    _, endpoint, report, dims = parts
+    report_type = ReportType(report.strip())
+    dimensions = validate_dimensions(dims)
+    endpoint = endpoint.strip()
+
+    return resource(
+        name="custom_report",
+        endpoint=endpoint,
+        dimensions=dimensions,
+        report_type=report_type,
+    )
+
+
+def validate_dimensions(dimensions: str) -> List[str]:
+    dims = [dim.strip() for dim in dimensions.split(",")]
+
+    if "day" not in dims:
+        dims.append("day")
+
+    return dims
+
+
+def exclude(source: List[str], exclude_list: List[str]) -> List[str]:
+    excludes = set(exclude_list)
+    return [col for col in source if col not in excludes]
+
+
+def build_type_hints(cols: List[str]) -> dict:
+    return {col: TYPE_HINTS[col] for col in cols if col in TYPE_HINTS}
+
+
+def http_error_handler(resp: Response):
+    if not resp.ok:
+        raise ClientError(f"HTTP Status {resp.status_code}: {resp.text}")

ingestr/src/applovin_max/__init__.py

@@ -0,0 +1,117 @@
+from datetime import timedelta
+from typing import Iterator
+
+import dlt
+import pandas as pd  # type: ignore[import-untyped]
+import pendulum
+import requests
+from dlt.sources import DltResource
+from dlt.sources.helpers.requests import Client
+from pendulum.date import Date
+
+
+@dlt.source(max_table_nesting=0)
+def applovin_max_source(
+    start_date: Date,
+    applications: list[str],
+    api_key: str,
+    end_date: Date | None,
+) -> DltResource:
+    @dlt.resource(
+        name="user_ad_revenue",
+        write_disposition="merge",
+        merge_key="partition_date",
+        columns={
+            "partition_date": {"data_type": "date", "partition": True},
+        },
+    )
+    def fetch_ad_revenue_report(
+        dateTime=(
+            dlt.sources.incremental(
+                "partition_date",
+                initial_value=start_date,
+                end_value=end_date,
+                range_start="closed",
+                range_end="closed",
+            )
+        ),
+    ) -> Iterator[dict]:
+        url = "https://r.applovin.com/max/userAdRevenueReport"
+        start_date = dateTime.last_value
+
+        if dateTime.end_value is None:
+            end_date = (pendulum.yesterday("UTC")).date()
+        else:
+            end_date = dateTime.end_value
+
+        client = create_client()
+        platforms = ["ios", "android", "fireos"]
+
+        for app in applications:
+            current_date = start_date
+            while current_date <= end_date:
+                for platform in platforms:
+                    df = get_data(
+                        url=url,
+                        current_date=current_date,
+                        application=app,
+                        api_key=api_key,
+                        client=client,
+                        platform=platform,
+                    )
+                    if df is not None:
+                        yield df
+                current_date = current_date + timedelta(days=1)
+
+    return fetch_ad_revenue_report
+
+
+def create_client() -> requests.Session:
+    return Client(
+        raise_for_status=False,
+        retry_condition=retry_on_limit,
+        request_max_attempts=12,
+    ).session
+
+
+def retry_on_limit(
+    response: requests.Response | None, exception: BaseException | None
+) -> bool:
+    if response is None:
+        return False
+    return response.status_code == 429
+
+
+def get_data(
+    url: str,
+    current_date: Date,
+    application: str,
+    api_key: str,
+    platform: str,
+    client: requests.Session,
+):
+    params = {
+        "api_key": api_key,
+        "date": current_date.isoformat(),
+        "platform": platform,
+        "application": application,
+        "aggregated": "false",
+    }
+
+    response = client.get(url=url, params=params)
+
+    if response.status_code != 200:
+        if response.status_code == 404:
+            if "No Mediation App Id found for platform" in response.text:
+                return None
+        error_message = (
+            f"AppLovin MAX API error (status {response.status_code}): {response.text}"
+        )
+        raise requests.HTTPError(error_message)
+
+    response_url = response.json().get("ad_revenue_report_url")
+    df = pd.read_csv(response_url)
+    df["Date"] = pd.to_datetime(df["Date"])
+    df["partition_date"] = df["Date"].dt.date
+    df["platform"] = platform
+    return df

ingestr/src/appsflyer/__init__.py

@@ -0,0 +1,325 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from ingestr.src.appsflyer.client import AppsflyerClient
+
+DIMENSION_RESPONSE_MAPPING = {
+    "c": "campaign",
+    "af_adset_id": "adset_id",
+    "af_adset": "adset",
+    "af_ad_id": "ad_id",
+}
+HINTS = {
+    "app_id": {
+        "data_type": "text",
+        "nullable": False,
+    },
+    "campaign": {
+        "data_type": "text",
+        "nullable": False,
+    },
+    "geo": {
+        "data_type": "text",
+        "nullable": False,
+    },
+    "cost": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": False,
+    },
+    "clicks": {
+        "data_type": "bigint",
+        "nullable": False,
+    },
+    "impressions": {
+        "data_type": "bigint",
+        "nullable": False,
+    },
+    "average_ecpi": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": False,
+    },
+    "installs": {
+        "data_type": "bigint",
+        "nullable": False,
+    },
+    "retention_day_7": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": False,
+    },
+    "retention_day_14": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": False,
+    },
+    "cohort_day_1_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_1_total_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_3_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_3_total_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_7_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_7_total_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_14_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_14_total_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_21_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_21_total_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "install_time": {
+        "data_type": "date",
+        "nullable": False,
+    },
+    "loyal_users": {
+        "data_type": "bigint",
+        "nullable": False,
+    },
+    "revenue": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "roi": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "uninstalls": {
+        "data_type": "bigint",
+        "nullable": True,
+    },
+}
+
+CAMPAIGNS_DIMENSIONS = ["c", "geo", "app_id", "install_time"]
+CAMPAIGNS_METRICS = [
+    "average_ecpi",
+    "clicks",
+    "cohort_day_1_revenue_per_user",
+    "cohort_day_1_total_revenue_per_user",
+    "cohort_day_14_revenue_per_user",
+    "cohort_day_14_total_revenue_per_user",
+    "cohort_day_21_revenue_per_user",
+    "cohort_day_21_total_revenue_per_user",
+    "cohort_day_3_revenue_per_user",
+    "cohort_day_3_total_revenue_per_user",
+    "cohort_day_7_revenue_per_user",
+    "cohort_day_7_total_revenue_per_user",
+    "cost",
+    "impressions",
+    "installs",
+    "loyal_users",
+    "retention_day_7",
+    "revenue",
+    "roi",
+    "uninstalls",
+]
+
+CREATIVES_DIMENSIONS = [
+    "c",
+    "geo",
+    "app_id",
+    "install_time",
+    "af_adset_id",
+    "af_adset",
+    "af_ad_id",
+]
+CREATIVES_METRICS = [
+    "impressions",
+    "clicks",
+    "installs",
+    "cost",
+    "revenue",
+    "average_ecpi",
+    "loyal_users",
+    "uninstalls",
+    "roi",
+]
+
+
+@dlt.source(max_table_nesting=0)
+def appsflyer_source(
+    api_key: str,
+    start_date: str,
+    end_date: str,
+    dimensions: list[str],
+    metrics: list[str],
+) -> Iterable[DltResource]:
+    client = AppsflyerClient(api_key)
+
+    @dlt.resource(
+        write_disposition="merge",
+        merge_key="install_time",
+        columns=make_hints(CAMPAIGNS_DIMENSIONS, CAMPAIGNS_METRICS),
+    )
+    def campaigns(
+        datetime=dlt.sources.incremental(
+            "install_time",
+            initial_value=(
+                start_date
+                if start_date
+                else pendulum.today().subtract(days=30).format("YYYY-MM-DD")
+            ),
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        end = (
+            datetime.end_value
+            if datetime.end_value
+            else pendulum.now().format("YYYY-MM-DD")
+        )
+
+        yield from client._fetch_data(
+            from_date=datetime.last_value,
+            to_date=end,
+            dimensions=CAMPAIGNS_DIMENSIONS,
+            metrics=CAMPAIGNS_METRICS,
+        )
+
+    @dlt.resource(
+        write_disposition="merge",
+        merge_key="install_time",
+        columns=make_hints(CREATIVES_DIMENSIONS, CREATIVES_METRICS),
+    )
+    def creatives(
+        datetime=dlt.sources.incremental(
+            "install_time",
+            initial_value=(
+                start_date
+                if start_date
+                else pendulum.today().subtract(days=30).format("YYYY-MM-DD")
+            ),
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        end = (
+            datetime.end_value
+            if datetime.end_value
+            else pendulum.now().format("YYYY-MM-DD")
+        )
+        yield from client._fetch_data(
+            datetime.last_value,
+            end,
+            dimensions=CREATIVES_DIMENSIONS,
+            metrics=CREATIVES_METRICS,
+        )
+
+    primary_keys = []
+    if "install_time" not in dimensions:
+        dimensions.append("install_time")
+        primary_keys.append("install_time")
+
+    for dimension in dimensions:
+        if dimension in DIMENSION_RESPONSE_MAPPING:
+            primary_keys.append(DIMENSION_RESPONSE_MAPPING[dimension])
+        else:
+            primary_keys.append(dimension)
+
+    @dlt.resource(
+        write_disposition="merge",
+        primary_key=primary_keys,
+        columns=make_hints(dimensions, metrics),
+    )
+    def custom(
+        datetime=dlt.sources.incremental(
+            "install_time",
+            initial_value=(
+                start_date
+                if start_date
+                else pendulum.today().subtract(days=30).format("YYYY-MM-DD")
+            ),
+            end_value=end_date,
+        ),
+    ):
+        end = (
+            datetime.end_value
+            if datetime.end_value
+            else pendulum.now().format("YYYY-MM-DD")
+        )
+        res = client._fetch_data(
+            from_date=datetime.last_value,
+            to_date=end,
+            dimensions=dimensions,
+            metrics=metrics,
+        )
+        yield from res
+
+    return campaigns, creatives, custom
+
+
+def make_hints(dimensions: list[str], metrics: list[str]):
+    campaign_hints = {}
+    for dimension in dimensions:
+        resp_key = dimension
+        if dimension in DIMENSION_RESPONSE_MAPPING:
+            resp_key = DIMENSION_RESPONSE_MAPPING[dimension]
+
+        if resp_key in HINTS:
+            campaign_hints[resp_key] = HINTS[resp_key]
+
+    for metric in metrics:
+        if metric in HINTS:
+            campaign_hints[metric] = HINTS[metric]
+
+    return campaign_hints