ingestr 0.13.1__tar.gz → 0.13.3__tar.gz

{ingestr-0.13.1 → ingestr-0.13.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.1
+Version: 0.13.3
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.1 → ingestr-0.13.3}/docs/.vitepress/config.mjs

@@ -96,6 +96,7 @@ export default defineConfig({
               { text: "Airtable", link: "/supported-sources/airtable.md" },
               { text: "AppsFlyer", link: "/supported-sources/appsflyer.md" },
               { text: "Apple App Store", link: "/supported-sources/appstore.md"},
+              { text: "Applovin", link: "/supported-sources/applovin.md"},
               { text: "Asana", link: "/supported-sources/asana.md" },
               { text: "Chess.com", link: "/supported-sources/chess.md" },
               { text: "DynamoDB", link: "/supported-sources/dynamodb.md" },

ingestr-0.13.3/docs/supported-sources/applovin.md

@@ -0,0 +1,144 @@
+# Applovin
+[AppLovin](https://www.applovin.com/) Corporation is an American mobile technology company headquartered in Palo Alto, California. AppLovin enables developers of all sizes to market, monetize, analyze and publish their apps through its mobile advertising, marketing, and analytics platforms MAX, AppDiscovery, and SparkLabs.
+
+`ingestr` allows ingesting data from [AppDiscovery reporting API](https://developers.applovin.com/en/app-discovery/api/reporting-api/)
+
+## URI Format
+
+The URI format for Applovin is as follows:
+```
+applovin://?api_key=<your_api_key>
+```
+
+URI Parameters:
+* `api_key`. report key generated from your [applovin account](https://www.applovin.com/analytics#keys).
+
+## Setting up Applovin Integration
+
+### Generate a Report Key
+You can generate a report key from your [analytics dashboard](https://www.applovin.com/analytics#keys).
+
+### Example: Loading Publisher Report
+
+For this example, we'll assume that:
+* `api_key` is `api_key_0`
+
+We will run `ingestr` to save this data to a [duckdb](https://duckdb.org/) database called `report.db` under the table name `public.publisher_report`.
+
+```sh
+ingestr ingest \
+    --source-uri "applovin://?api_key=api_key_0
+    --source-table "publisher-report" \
+    --dest-uri "duckdb:///report.db"  \
+    --dest-table "public.publisher_report" 
+```
+
+### Example: Incremental loading
+
+We will extend the [Loading Publisher Report](#example-loading-publisher-report) example to demonstrate incremental loading.
+
+First, we run the example with a start date of `2025-01-01` and an end date of `2025-01-05`
+
+```sh
+ingestr ingest \
+    --source-uri "applovin://?api_key=api_key_0
+    --source-table "publisher-report" \
+    --dest-uri "duckdb:///report.db"  \
+    --dest-table "public.publisher_report" \
+    --interval-start "2025-01-01" \
+    --interval-end "2025-01-05" 
+```
+
+We can query the database to see which dates the data was ingested for:
+
+```sh
+$ duckdb report.db 'select day from public.publisher_report group by 1'
+┌────────────┐
+│    day     │
+│    date    │
+├────────────┤
+│ 2025-01-01 │
+│ 2025-01-02 │
+│ 2025-01-03 │
+│ 2025-01-04 │
+│ 2025-01-05 │
+└────────────┘
+```
+Now, we will run `ingestr` again, but we will omit start and end date to demonstrate an incremental load.
+
+```sh
+ingestr ingest \
+    --source-uri "applovin://?api_key=api_key_0
+    --source-table "publisher-report" \
+    --dest-uri "duckdb:///report.db"  \
+    --dest-table "public.publisher_report" 
+```
+
+Now we can check the database again, and we will see that the data was loaded for the rest of the days automatically.
+```sh
+$ duckdb report.db 'select day from public.publisher_report group by 1'
+┌────────────┐
+│    day     │
+│    date    │
+├────────────┤
+│ 2025-01-01 │
+│ 2025-01-02 │
+│ 2025-01-03 │
+│ 2025-01-04 │
+│ 2025-01-05 │
+│ 2025-01-06 │
+│     .      │
+│     .      │
+│     .      │
+│ 2025-01-28 │
+│ 2025-01-29 │
+│ 2025-01-30 │
+│ 2025-01-31 │
+├────────────┤
+│  35 rows   │
+└────────────┘
+```
+
+## Tables
+
+| Name | Description |
+| --- | --- |
+| `publisher-report` | Provides daily metrics from the `report` end point using the report_type `publisher` |
+| `advertiser-report` | Provides daily metrics from the `report` end point using the report_type `advertiser`|
+| `advertiser-probabilistic-report` | Provides daily metrics from the `probabilisticReport` end point using the report_type `advertiser` |
+| `advertiser-ska-report` | Provides daily metrics from the `skaReport` end point using the report_type `advertiser` |
+
+## Custom Reports
+
+`applovin` source supports custom reports. You can pass a custom report definition to `--source-table` and it will dynamically create a report for you.
+
+The format of a custom report looks like the following:
+```
+custom:{endpoint}:{report_type}:{columns}
+```
+Where:
+* `{endpoint}` is the API endpoint for applovin reports (one of `report`, `probabilisticReport` or `skaReport`)
+* `{report_type}` is the [report type](https://developers.applovin.com/en/app-discovery/api/reporting-api#required-parameters) (one of `publisher` or `advertiser`)
+* `{columns}` are the [columns](https://developers.applovin.com/en/app-discovery/api/reporting-api#allowed-publisher-columns) of the given report type.
+
+
+### Custom Report Example
+For this example, we will ingest data from `report` end point with the report type `publisher`.
+
+We want to obtain the following columns:
+* ad_type
+* clicks
+* country
+
+To achieve this, we can pass the custom report defintion in `--source-table`
+
+```sh
+ingestr ingest \
+    --source-uri "applovin://?api_key=api_key_0
+    --source-table "custom:report:publisher:ad_type,clicks,country" \
+    --dest-uri "duckdb:///report.db"  \
+    --dest-table "public.custom_report" 
+```
+
+> [!NOTE]
+> The `day` column is automatically added to any custom report if it is not specified in the custom report definition.

{ingestr-0.13.1 → ingestr-0.13.3}/docs/supported-sources/clickhouse.md

@@ -7,14 +7,15 @@ Ingestr supports ClickHouse as a destination.
 The URI format for ClickHouse as a destination is as follows:
 
 ```plaintext
-clickhouse://<username>:<password>@<host>:<port>?http_port=<http_port>
+clickhouse://<username>:<password>@<host>:<port>?http_port=<http_port>&secure=<secure>
 ```
 ## URI parameters:
 - `username` (required): The username is required to authenticate with the ClickHouse server.
 - `password` (required): The password is required to authenticate the provided username.
 - `host` (required): The hostname or IP address of the ClickHouse server where the database is hosted.
 - `port` (required): The TCP port number used by the ClickHouse server.
-- `http_port` (optional): The port number to use when connecting to the ClickHouse server's HTTP interface. Make sure your ClickHouse server is configured to accept HTTP connections on the port specified by http_port. By default, ClickHouse uses port 8123.
+- `http_port` (optional): The port number to use when connecting to the ClickHouse server's HTTP interface.By default, it is set to port `8443`.
+- `secure` (optional): Set to `1` for a secure HTTPS connection or `0` for a non-secure HTTP connection. By default, it is set to `1`.
 
 ClickHouse requires a `username`, `password`, `host` and `port` to connect to the ClickHouse server. For more information, read [here](https://dlthub.com/docs/dlt-ecosystem/destinations/clickhouse#2-setup-clickhouse-database). Once you've completed the guide, you should have all the above-mentioned credentials.
 

ingestr-0.13.3/ingestr/src/applovin/__init__.py

@@ -0,0 +1,282 @@
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Dict, List, Optional
+
+import dlt
+from dlt.sources.rest_api import EndpointResource, RESTAPIConfig, rest_api_resources
+
+
+class InvalidCustomReportError(Exception):
+    def __init__(self):
+        super().__init__(
+            "Custom report should be in the format 'custom:{endpoint}:{report_type}:{dimensions}"
+        )
+
+
+class InvalidDimensionError(Exception):
+    def __init__(self, dim: str, report_type: str):
+        super().__init__(f"Unknown dimension {dim} for report type {report_type}")
+
+
+TYPE_HINTS = {
+    "application_is_hidden": {"data_type": "bool"},
+    "average_cpa": {"data_type": "double"},
+    "average_cpc": {"data_type": "double"},
+    "campaign_bid_goal": {"data_type": "double"},
+    "campaign_roas_goal": {"data_type": "double"},
+    "clicks": {"data_type": "bigint"},
+    "conversions": {"data_type": "bigint"},
+    "conversion_rate": {"data_type": "double"},
+    "cost": {"data_type": "double"},  # assuming float.
+    "ctr": {"data_type": "double"},
+    "day": {"data_type": "date"},
+    "first_purchase": {"data_type": "bigint"},
+    "ecpm": {"data_type": "double"},
+    "impressions": {"data_type": "bigint"},
+    "installs": {"data_type": "bigint"},
+    "revenue": {"data_type": "double"},
+    "redownloads": {"data_type": "bigint"},
+    "sales": {"data_type": "double"},  # assuming float.
+}
+
+
+class ReportType(Enum):
+    PUBLISHER = "publisher"
+    ADVERTISER = "advertiser"
+
+
+REPORT_SCHEMA: Dict[ReportType, List[str]] = {
+    ReportType.PUBLISHER: [
+        "ad_type",
+        "application",
+        "application_is_hidden",
+        "bidding_integration",
+        "clicks",
+        "country",
+        "ctr",
+        "day",
+        "device_type",
+        "ecpm",
+        "impressions",
+        "package_name",
+        "placement_type",
+        "platform",
+        "revenue",
+        "size",
+        "store_id",
+        "zone",
+        "zone_id",
+    ],
+    ReportType.ADVERTISER: [
+        "ad",
+        "ad_creative_type",
+        "ad_id",
+        "ad_type",
+        "average_cpa",
+        "average_cpc",
+        "campaign",
+        "campaign_ad_type",
+        "campaign_bid_goal",
+        "campaign_id_external",
+        "campaign_package_name",
+        "campaign_roas_goal",
+        "campaign_store_id",
+        "campaign_type",
+        "clicks",
+        "conversions",
+        "conversion_rate",
+        "cost",
+        "country",
+        "creative_set",
+        "creative_set_id",
+        "ctr",
+        "custom_page_id",
+        "day",
+        "device_type",
+        "external_placement_id",
+        "first_purchase",
+        "impressions",
+        "installs",
+        "optimization_day_target",
+        "placement_type",
+        "platform",
+        "redownloads",
+        "sales",
+        "size",
+        "target_event",
+        "traffic_source",
+    ],
+}
+
+# NOTE(turtledev): These values are valid columns,
+# but often don't produce a value. Find a way to either add
+# a default value, or use an alternative strategy to de-duplicate
+# OR make them nullable
+SKA_REPORT_EXCLUDE = [
+    "ad",
+    "ad_id",
+    "ad_type",
+    "average_cpc",
+    "campaign_ad_type",
+    "clicks",
+    "conversions",
+    "conversion_rate",
+    "creative_set",
+    "creative_set_id",
+    "ctr",
+    "custom_page_id",
+    "device_type",
+    "first_purchase",
+    "impressions",
+    "placement_type",
+    "sales",
+    "size",
+    "traffic_source",
+]
+
+PROBABILISTIC_REPORT_EXCLUDE = [
+    "installs",
+    "redownloads",
+]
+
+
+@dlt.source
+def applovin_source(
+    api_key: str,
+    start_date: str,
+    end_date: Optional[str],
+    custom: Optional[str],
+):
+    ska_report_columns = exclude(
+        REPORT_SCHEMA[ReportType.ADVERTISER],
+        SKA_REPORT_EXCLUDE,
+    )
+
+    probabilistic_report_columns = exclude(
+        REPORT_SCHEMA[ReportType.ADVERTISER],
+        PROBABILISTIC_REPORT_EXCLUDE,
+    )
+    backfill = False
+    if end_date is None:
+        backfill = True
+        end_date = datetime.now(timezone.utc).date().strftime("%Y-%m-%d")
+
+    config: RESTAPIConfig = {
+        "client": {
+            "base_url": "https://r.applovin.com/",
+            "auth": {
+                "type": "api_key",
+                "name": "api_key",
+                "location": "query",
+                "api_key": api_key,
+            },
+        },
+        "resource_defaults": {
+            "write_disposition": "merge",
+            "endpoint": {
+                "incremental": {
+                    "cursor_path": "day",
+                    "start_param": "start",
+                    "initial_value": start_date,
+                    "range_start": "closed",
+                    "range_end": "closed",
+                },
+                "params": {
+                    "format": "json",
+                    "end": end_date,
+                },
+                "paginator": "single_page",
+            },
+        },
+        "resources": [
+            resource(
+                "publisher-report",
+                "report",
+                REPORT_SCHEMA[ReportType.PUBLISHER],
+                ReportType.PUBLISHER,
+            ),
+            resource(
+                "advertiser-report",
+                "report",
+                REPORT_SCHEMA[ReportType.ADVERTISER],
+                ReportType.ADVERTISER,
+            ),
+            resource(
+                "advertiser-probabilistic-report",
+                "probabilisticReport",
+                probabilistic_report_columns,
+                ReportType.ADVERTISER,
+            ),
+            resource(
+                "advertiser-ska-report",
+                "skaReport",
+                ska_report_columns,
+                ReportType.ADVERTISER,
+            ),
+        ],
+    }
+
+    if custom:
+        custom_report = custom_report_from_spec(custom)
+        config["resources"].append(custom_report)
+
+    if backfill:
+        config["resource_defaults"]["endpoint"]["incremental"]["end_value"] = end_date  # type: ignore
+
+    yield from rest_api_resources(config)
+
+
+def resource(
+    name: str,
+    endpoint: str,
+    dimensions: List[str],
+    report_type: ReportType,
+) -> EndpointResource:
+    return {
+        "name": name,
+        "columns": build_type_hints(dimensions),
+        "merge_key": "day",
+        "endpoint": {
+            "path": endpoint,
+            "params": {
+                "report_type": report_type.value,
+                "columns": ",".join(dimensions),
+            },
+        },
+    }
+
+
+def custom_report_from_spec(spec: str) -> EndpointResource:
+    parts = spec.split(":")
+    if len(parts) != 4:
+        raise InvalidCustomReportError()
+
+    _, endpoint, report, dims = parts
+    report_type = ReportType(report.strip())
+    dimensions = validate_dimensions(dims)
+    endpoint = endpoint.strip()
+
+    return resource(
+        name="custom_report",
+        endpoint=endpoint,
+        dimensions=dimensions,
+        report_type=report_type,
+    )
+
+
+def validate_dimensions(dimensions: str) -> List[str]:
+    dims = [dim.strip() for dim in dimensions.split(",")]
+
+    if "day" not in dims:
+        dims.append("day")
+
+    return dims
+
+
+def exclude(source: List[str], exclude_list: List[str]) -> List[str]:
+    excludes = set(exclude_list)
+    return [col for col in source if col not in excludes]
+
+
+def build_type_hints(cols: List[str]) -> dict:
+    return {col: TYPE_HINTS[col] for col in cols if col in TYPE_HINTS}

{ingestr-0.13.1 → ingestr-0.13.3}/ingestr/src/destinations.py

@@ -299,10 +299,21 @@ class ClickhouseDestination:
             )
 
         query_params = parse_qs(parsed_uri.query)
+        secure = int(query_params["secure"][0]) if "secure" in query_params else 1
+
         http_port = (
-            int(query_params["http_port"][0]) if "http_port" in query_params else 8123
+            int(query_params["http_port"][0])
+            if "http_port" in query_params
+            else 8443
+            if secure == 1
+            else 8123
         )
 
+        if secure not in (0, 1):
+            raise ValueError(
+                "Invalid value for secure. Set to `1` for a secure HTTPS connection or `0` for a non-secure HTTP connection."
+            )
+
         credentials = ClickHouseCredentials(
             {
                 "host": host,
@@ -311,10 +322,9 @@ class ClickhouseDestination:
                 "password": password,
                 "database": database,
                 "http_port": http_port,
-                "secure": 0,
+                "secure": secure,
             }
         )
-
         return dlt.destinations.clickhouse(credentials=credentials)
 
     def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:

{ingestr-0.13.1 → ingestr-0.13.3}/ingestr/src/factory.py

@@ -20,6 +20,7 @@ from ingestr.src.sources import (
     AdjustSource,
     AirtableSource,
     AppleAppStoreSource,
+    AppLovinSource,
     AppsflyerSource,
     ArrowMemoryMappedSource,
     AsanaSource,
@@ -131,6 +132,7 @@ class SourceDestinationFactory:
         "appstore": AppleAppStoreSource,
         "gs": GCSSource,
         "linkedinads": LinkedInAdsSource,
+        "applovin": AppLovinSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

{ingestr-0.13.1 → ingestr-0.13.3}/ingestr/src/sources.py

@@ -50,6 +50,7 @@ from ingestr.src import blob
 from ingestr.src.adjust import REQUIRED_CUSTOM_DIMENSIONS, adjust_source
 from ingestr.src.adjust.adjust_helpers import parse_filters
 from ingestr.src.airtable import airtable_source
+from ingestr.src.applovin import applovin_source
 from ingestr.src.appsflyer._init_ import appsflyer_source
 from ingestr.src.appstore import app_store
 from ingestr.src.appstore.client import AppStoreConnectClient
@@ -1737,3 +1738,47 @@ class LinkedInAdsSource:
             metrics=metrics,
             time_granularity=time_granularity,
         ).with_resources("custom_reports")
+
+
+class AppLovinSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key") is not None:
+            raise ValueError(
+                "Google Ads takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+
+        api_key = params.get("api_key", None)
+        if api_key is None:
+            raise MissingValueError("api_key", "AppLovin")
+
+        interval_start = kwargs.get("interval_start")
+        interval_end = kwargs.get("interval_end")
+
+        now = datetime.now()
+        start_date = (
+            interval_start if interval_start is not None else now - timedelta(days=1)
+        )
+        end_date = interval_end
+
+        custom_report = None
+        if table.startswith("custom:"):
+            custom_report = table
+            table = "custom_report"
+
+        src = applovin_source(
+            api_key[0],
+            start_date.strftime("%Y-%m-%d"),
+            end_date.strftime("%Y-%m-%d") if end_date else None,
+            custom_report,
+        )
+
+        if table not in src.resources:
+            raise UnsupportedResourceError(table, "AppLovin")
+
+        return src.with_resources(table)

ingestr-0.13.3/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.13.3"

ingestr-0.13.1/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.13.1"