ingestr 0.13.5__tar.gz → 0.13.7__tar.gz

{ingestr-0.13.5 → ingestr-0.13.7}/.github/workflows/release.yml

@@ -1,3 +1,5 @@
+# WARN: this is used as a notification filter in our slack.
+# When updating this, make sure to update the workflow subscription as well.
 name: "Release ingestr on pip and ghcr.io"
 
 on:

{ingestr-0.13.5 → ingestr-0.13.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.5
+Version: 0.13.7
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.5 → ingestr-0.13.7}/docs/.vitepress/config.mjs

@@ -97,6 +97,7 @@ export default defineConfig({
               { text: "AppsFlyer", link: "/supported-sources/appsflyer.md" },
               { text: "Apple App Store", link: "/supported-sources/appstore.md"},
               { text: "Applovin", link: "/supported-sources/applovin.md"},
+              { text: "Applovin Max", link: "/supported-sources/applovin_max.md"},
               { text: "Asana", link: "/supported-sources/asana.md" },
               { text: "Chess.com", link: "/supported-sources/chess.md" },
               { text: "DynamoDB", link: "/supported-sources/dynamodb.md" },

ingestr-0.13.7/docs/supported-sources/applovin_max.md

@@ -0,0 +1,36 @@
+# Applovin Max
+[AppLovin Max](https://developers.applovin.com/en/max/getting-started/) is a tool from AppLovin that helps app developers optimize ad revenue by selecting the highest-paying ads from various ad networks.
+
+`ingestr` allows ingesting data from AppLovin Max reporting API.
+
+## URI Format
+The URI format for Applovin Max is as follows:
+```
+applovinmax://?api_key=<your_api_key>&application=<application_name>
+```
+
+URI Parameters:
+- `api_key`: It is the `report key` which is used for authenticating the request.
+- `application`: The application package name (for Android and Fire OS) or bundle ID (for iOS).
+
+## Setting up Applovin Integration
+
+### Generate a Report Key
+You can generate a report key from your [AppLovin dashboard](https://developers.applovin.com/en/max/max-dashboard/account/account-info/#keys).
+
+### Example:
+To retrieve `ad revenue` data for an application `com.example.app` with api key `key_123`, and to store it in a DuckDB database, use the following command:
+
+```sh
+ingestr ingest \
+    --source-uri "applovinmax://?api_key=key_123&application=com.example.app" \
+    --source-table "ad_revenue" \
+    --dest-uri "duckdb:///applovin_max.db"  \
+    --dest-table "dest.ad_revenue"
+```
+By default, `ingestr` retrieves data for the last 30 days. For a custom date range, use the `--interval-start` and `--interval-end` parameters.
+
+<img alt="applovin_max_img" src="../media/applovin_max.png"/>
+
+## Table
+[ad_revenue](https://developers.applovin.com/en/max/reporting-apis/user-level-ad-revenue-api/): Provides daily metrics from the user-level ad revenue API.User-level revenue data is available eight hours after UTC day end. So, for example, data for UTC 2025-01-01 is available on UTC 2025-01-02 after 08:00.

{ingestr-0.13.5 → ingestr-0.13.7}/ingestr/src/applovin/__init__.py

@@ -1,6 +1,7 @@
-from datetime import datetime, timezone
+from datetime import datetime, timezone, timedelta
 from enum import Enum
 from typing import Dict, List, Optional
+from requests import Response
 
 import dlt
 from dlt.sources.rest_api import EndpointResource, RESTAPIConfig, rest_api_resources
@@ -12,11 +13,8 @@ class InvalidCustomReportError(Exception):
             "Custom report should be in the format 'custom:{endpoint}:{report_type}:{dimensions}"
         )
 
-
-class InvalidDimensionError(Exception):
-    def __init__(self, dim: str, report_type: str):
-        super().__init__(f"Unknown dimension {dim} for report type {report_type}")
-
+class ClientError(Exception):
+    pass
 
 TYPE_HINTS = {
     "application_is_hidden": {"data_type": "bool"},
@@ -108,32 +106,6 @@ REPORT_SCHEMA: Dict[ReportType, List[str]] = {
     ],
 }
 
-# NOTE(turtledev): These values are valid columns,
-# but often don't produce a value. Find a way to either add
-# a default value, or use an alternative strategy to de-duplicate
-# OR make them nullable
-SKA_REPORT_EXCLUDE = [
-    "ad",
-    "ad_id",
-    "ad_type",
-    "average_cpc",
-    "campaign_ad_type",
-    "clicks",
-    "conversions",
-    "conversion_rate",
-    "creative_set",
-    "creative_set_id",
-    "ctr",
-    "custom_page_id",
-    "device_type",
-    "first_purchase",
-    "impressions",
-    "placement_type",
-    "sales",
-    "size",
-    "traffic_source",
-]
-
 PROBABILISTIC_REPORT_EXCLUDE = [
     "installs",
     "redownloads",
@@ -147,19 +119,16 @@ def applovin_source(
     end_date: Optional[str],
     custom: Optional[str],
 ):
-    ska_report_columns = exclude(
-        REPORT_SCHEMA[ReportType.ADVERTISER],
-        SKA_REPORT_EXCLUDE,
-    )
 
-    probabilistic_report_columns = exclude(
-        REPORT_SCHEMA[ReportType.ADVERTISER],
-        PROBABILISTIC_REPORT_EXCLUDE,
-    )
     backfill = False
     if end_date is None:
         backfill = True
-        end_date = datetime.now(timezone.utc).date().strftime("%Y-%m-%d")
+
+        # use the greatest of yesterday and start_date
+        end_date = max(
+            datetime.now(timezone.utc) - timedelta(days=1),
+            datetime.fromisoformat(start_date).replace(tzinfo=timezone.utc)
+        ).strftime("%Y-%m-%d")
 
     config: RESTAPIConfig = {
         "client": {
@@ -186,6 +155,9 @@ def applovin_source(
                     "end": end_date,
                 },
                 "paginator": "single_page",
+                "response_actions": [
+                    http_error_handler,
+                ]
             },
         },
         "resources": [
@@ -204,13 +176,16 @@ def applovin_source(
             resource(
                 "advertiser-probabilistic-report",
                 "probabilisticReport",
-                probabilistic_report_columns,
+                exclude(
+                    REPORT_SCHEMA[ReportType.ADVERTISER],
+                    PROBABILISTIC_REPORT_EXCLUDE
+                ),
                 ReportType.ADVERTISER,
             ),
             resource(
                 "advertiser-ska-report",
                 "skaReport",
-                ska_report_columns,
+                REPORT_SCHEMA[ReportType.ADVERTISER],
                 ReportType.ADVERTISER,
             ),
         ],
@@ -280,3 +255,7 @@ def exclude(source: List[str], exclude_list: List[str]) -> List[str]:
 
 def build_type_hints(cols: List[str]) -> dict:
     return {col: TYPE_HINTS[col] for col in cols if col in TYPE_HINTS}
+
+def http_error_handler(resp: Response):
+    if not resp.ok:
+        raise ClientError(f"HTTP Status {resp.status_code}: {resp.text}")

ingestr-0.13.7/ingestr/src/applovin_max/__init__.py

@@ -0,0 +1,99 @@
+from typing import Iterator
+
+import dlt
+import pandas as pd  # type: ignore[import-untyped]
+import pendulum
+import requests
+from dlt.sources import DltResource
+from dlt.sources.helpers.requests import Client
+from pendulum.date import Date
+
+
+@dlt.source(max_table_nesting=0)
+def applovin_max_source(
+    start_date: str,
+    application: str,
+    api_key: str,
+    end_date: str | None,
+) -> DltResource:
+    @dlt.resource(
+        name="ad_revenue",
+        write_disposition="merge",
+        merge_key="_partition_date",
+    )
+    def fetch_ad_revenue_report(
+        dateTime=(
+            dlt.sources.incremental(
+                "_partition_date",
+                initial_value=start_date,
+                end_value=end_date,
+                range_start="closed",
+                range_end="closed",
+            )
+        ),
+    ) -> Iterator[dict]:
+        url = "https://r.applovin.com/max/userAdRevenueReport"
+        start_date = pendulum.from_format(dateTime.last_value, "YYYY-MM-DD").date()
+        if dateTime.end_value is None:
+            end_date = (pendulum.yesterday("UTC")).date()
+        else:
+            end_date = pendulum.from_format(dateTime.end_value, "YYYY-MM-DD").date()
+        yield get_data(
+            url=url,
+            start_date=start_date,
+            end_date=end_date,
+            application=application,
+            api_key=api_key,
+        )
+
+    return fetch_ad_revenue_report
+
+
+def create_client() -> requests.Session:
+    return Client(
+        request_timeout=10.0,
+        raise_for_status=False,
+        retry_condition=retry_on_limit,
+        request_max_attempts=12,
+    ).session
+
+
+def retry_on_limit(
+    response: requests.Response | None, exception: BaseException | None
+) -> bool:
+    if response is None:
+        return False
+    return response.status_code == 429
+
+
+def get_data(
+    url: str, start_date: Date, end_date: Date, application: str, api_key: str
+):
+    client = create_client()
+    platforms = ["ios", "android", "fireos"]
+    current_date = start_date
+    while current_date <= end_date:
+        for platform in platforms:
+            params = {
+                "api_key": api_key,
+                "date": current_date.strftime("%Y-%m-%d"),
+                "platform": platform,
+                "application": application,
+                "aggregated": "false",
+            }
+
+            response = client.get(url=url, params=params)
+
+            if response.status_code == 400:
+                raise ValueError(response.text)
+
+            if response.status_code != 200:
+                continue
+
+            response_url = response.json().get("ad_revenue_report_url")
+            df = pd.read_csv(response_url)
+            df["Date"] = pd.to_datetime(df["Date"])
+            df["_partition_date"] = df["Date"].dt.strftime("%Y-%m-%d")
+            yield df
+
+        current_date = current_date.add(days=1)

{ingestr-0.13.5 → ingestr-0.13.7}/ingestr/src/blob.py

@@ -26,7 +26,7 @@ def parse_uri(uri: ParseResult, table: str) -> Tuple[BucketName, FileGlob]:
     table = table.strip()
     host = uri.netloc.strip()
 
-    if table == "":
+    if table == "" or uri.path.strip() != "":
         warnings.warn(
             f"Using the form '{uri.scheme}://bucket-name/file-glob' is deprecated and will be removed in future versions.",
             DeprecationWarning,

{ingestr-0.13.5 → ingestr-0.13.7}/ingestr/src/factory.py

@@ -20,6 +20,7 @@ from ingestr.src.sources import (
     AdjustSource,
     AirtableSource,
     AppleAppStoreSource,
+    ApplovinMaxSource,
     AppLovinSource,
     AppsflyerSource,
     ArrowMemoryMappedSource,
@@ -134,6 +135,7 @@ class SourceDestinationFactory:
         "gs": GCSSource,
         "linkedinads": LinkedInAdsSource,
         "applovin": AppLovinSource,
+        "applovinmax": ApplovinMaxSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

{ingestr-0.13.5 → ingestr-0.13.7}/ingestr/src/sources.py

@@ -51,6 +51,7 @@ from ingestr.src.adjust import REQUIRED_CUSTOM_DIMENSIONS, adjust_source
 from ingestr.src.adjust.adjust_helpers import parse_filters
 from ingestr.src.airtable import airtable_source
 from ingestr.src.applovin import applovin_source
+from ingestr.src.applovin_max import applovin_max_source
 from ingestr.src.appsflyer._init_ import appsflyer_source
 from ingestr.src.appstore import app_store
 from ingestr.src.appstore.client import AppStoreConnectClient
@@ -1787,3 +1788,48 @@ class AppLovinSource:
             raise UnsupportedResourceError(table, "AppLovin")
 
         return src.with_resources(table)
+
+
+class ApplovinMaxSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+
+        api_key = params.get("api_key")
+        if api_key is None:
+            raise ValueError("api_key is required to connect to AppLovin Max API.")
+
+        application = params.get("application")
+        if application is None:
+            raise ValueError("application is required to connect to AppLovin Max API.")
+
+        interval_start = kwargs.get("interval_start")
+        interval_end = kwargs.get("interval_end")
+
+        if "ad_revenue" in table:
+            table = "ad_revenue"
+        else:
+            raise ValueError(
+                f"Table name '{table}' is not supported for AppLovin Max source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+            )
+
+        now = pendulum.now("UTC")
+        default_start = now.subtract(days=30).date()
+
+        start_date = (
+            interval_start if interval_start is not None else default_start
+        ).strftime("%Y-%m-%d")
+
+        end_date = (
+            interval_end.strftime("%Y-%m-%d") if interval_end is not None else None
+        )
+
+        return applovin_max_source(
+            start_date=start_date,
+            end_date=end_date,
+            api_key=api_key[0],
+            application=application[0],
+        ).with_resources(table)