ingestr 0.13.1__py3-none-any.whl → 0.13.3__py3-none-any.whl

ingestr/src/applovin/__init__.py

@@ -0,0 +1,282 @@
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Dict, List, Optional
+
+import dlt
+from dlt.sources.rest_api import EndpointResource, RESTAPIConfig, rest_api_resources
+
+
+class InvalidCustomReportError(Exception):
+    def __init__(self):
+        super().__init__(
+            "Custom report should be in the format 'custom:{endpoint}:{report_type}:{dimensions}"
+        )
+
+
+class InvalidDimensionError(Exception):
+    def __init__(self, dim: str, report_type: str):
+        super().__init__(f"Unknown dimension {dim} for report type {report_type}")
+
+
+TYPE_HINTS = {
+    "application_is_hidden": {"data_type": "bool"},
+    "average_cpa": {"data_type": "double"},
+    "average_cpc": {"data_type": "double"},
+    "campaign_bid_goal": {"data_type": "double"},
+    "campaign_roas_goal": {"data_type": "double"},
+    "clicks": {"data_type": "bigint"},
+    "conversions": {"data_type": "bigint"},
+    "conversion_rate": {"data_type": "double"},
+    "cost": {"data_type": "double"},  # assuming float.
+    "ctr": {"data_type": "double"},
+    "day": {"data_type": "date"},
+    "first_purchase": {"data_type": "bigint"},
+    "ecpm": {"data_type": "double"},
+    "impressions": {"data_type": "bigint"},
+    "installs": {"data_type": "bigint"},
+    "revenue": {"data_type": "double"},
+    "redownloads": {"data_type": "bigint"},
+    "sales": {"data_type": "double"},  # assuming float.
+}
+
+
+class ReportType(Enum):
+    PUBLISHER = "publisher"
+    ADVERTISER = "advertiser"
+
+
+REPORT_SCHEMA: Dict[ReportType, List[str]] = {
+    ReportType.PUBLISHER: [
+        "ad_type",
+        "application",
+        "application_is_hidden",
+        "bidding_integration",
+        "clicks",
+        "country",
+        "ctr",
+        "day",
+        "device_type",
+        "ecpm",
+        "impressions",
+        "package_name",
+        "placement_type",
+        "platform",
+        "revenue",
+        "size",
+        "store_id",
+        "zone",
+        "zone_id",
+    ],
+    ReportType.ADVERTISER: [
+        "ad",
+        "ad_creative_type",
+        "ad_id",
+        "ad_type",
+        "average_cpa",
+        "average_cpc",
+        "campaign",
+        "campaign_ad_type",
+        "campaign_bid_goal",
+        "campaign_id_external",
+        "campaign_package_name",
+        "campaign_roas_goal",
+        "campaign_store_id",
+        "campaign_type",
+        "clicks",
+        "conversions",
+        "conversion_rate",
+        "cost",
+        "country",
+        "creative_set",
+        "creative_set_id",
+        "ctr",
+        "custom_page_id",
+        "day",
+        "device_type",
+        "external_placement_id",
+        "first_purchase",
+        "impressions",
+        "installs",
+        "optimization_day_target",
+        "placement_type",
+        "platform",
+        "redownloads",
+        "sales",
+        "size",
+        "target_event",
+        "traffic_source",
+    ],
+}
+
+# NOTE(turtledev): These values are valid columns,
+# but often don't produce a value. Find a way to either add
+# a default value, or use an alternative strategy to de-duplicate
+# OR make them nullable
+SKA_REPORT_EXCLUDE = [
+    "ad",
+    "ad_id",
+    "ad_type",
+    "average_cpc",
+    "campaign_ad_type",
+    "clicks",
+    "conversions",
+    "conversion_rate",
+    "creative_set",
+    "creative_set_id",
+    "ctr",
+    "custom_page_id",
+    "device_type",
+    "first_purchase",
+    "impressions",
+    "placement_type",
+    "sales",
+    "size",
+    "traffic_source",
+]
+
+PROBABILISTIC_REPORT_EXCLUDE = [
+    "installs",
+    "redownloads",
+]
+
+
+@dlt.source
+def applovin_source(
+    api_key: str,
+    start_date: str,
+    end_date: Optional[str],
+    custom: Optional[str],
+):
+    ska_report_columns = exclude(
+        REPORT_SCHEMA[ReportType.ADVERTISER],
+        SKA_REPORT_EXCLUDE,
+    )
+
+    probabilistic_report_columns = exclude(
+        REPORT_SCHEMA[ReportType.ADVERTISER],
+        PROBABILISTIC_REPORT_EXCLUDE,
+    )
+    backfill = False
+    if end_date is None:
+        backfill = True
+        end_date = datetime.now(timezone.utc).date().strftime("%Y-%m-%d")
+
+    config: RESTAPIConfig = {
+        "client": {
+            "base_url": "https://r.applovin.com/",
+            "auth": {
+                "type": "api_key",
+                "name": "api_key",
+                "location": "query",
+                "api_key": api_key,
+            },
+        },
+        "resource_defaults": {
+            "write_disposition": "merge",
+            "endpoint": {
+                "incremental": {
+                    "cursor_path": "day",
+                    "start_param": "start",
+                    "initial_value": start_date,
+                    "range_start": "closed",
+                    "range_end": "closed",
+                },
+                "params": {
+                    "format": "json",
+                    "end": end_date,
+                },
+                "paginator": "single_page",
+            },
+        },
+        "resources": [
+            resource(
+                "publisher-report",
+                "report",
+                REPORT_SCHEMA[ReportType.PUBLISHER],
+                ReportType.PUBLISHER,
+            ),
+            resource(
+                "advertiser-report",
+                "report",
+                REPORT_SCHEMA[ReportType.ADVERTISER],
+                ReportType.ADVERTISER,
+            ),
+            resource(
+                "advertiser-probabilistic-report",
+                "probabilisticReport",
+                probabilistic_report_columns,
+                ReportType.ADVERTISER,
+            ),
+            resource(
+                "advertiser-ska-report",
+                "skaReport",
+                ska_report_columns,
+                ReportType.ADVERTISER,
+            ),
+        ],
+    }
+
+    if custom:
+        custom_report = custom_report_from_spec(custom)
+        config["resources"].append(custom_report)
+
+    if backfill:
+        config["resource_defaults"]["endpoint"]["incremental"]["end_value"] = end_date  # type: ignore
+
+    yield from rest_api_resources(config)
+
+
+def resource(
+    name: str,
+    endpoint: str,
+    dimensions: List[str],
+    report_type: ReportType,
+) -> EndpointResource:
+    return {
+        "name": name,
+        "columns": build_type_hints(dimensions),
+        "merge_key": "day",
+        "endpoint": {
+            "path": endpoint,
+            "params": {
+                "report_type": report_type.value,
+                "columns": ",".join(dimensions),
+            },
+        },
+    }
+
+
+def custom_report_from_spec(spec: str) -> EndpointResource:
+    parts = spec.split(":")
+    if len(parts) != 4:
+        raise InvalidCustomReportError()
+
+    _, endpoint, report, dims = parts
+    report_type = ReportType(report.strip())
+    dimensions = validate_dimensions(dims)
+    endpoint = endpoint.strip()
+
+    return resource(
+        name="custom_report",
+        endpoint=endpoint,
+        dimensions=dimensions,
+        report_type=report_type,
+    )
+
+
+def validate_dimensions(dimensions: str) -> List[str]:
+    dims = [dim.strip() for dim in dimensions.split(",")]
+
+    if "day" not in dims:
+        dims.append("day")
+
+    return dims
+
+
+def exclude(source: List[str], exclude_list: List[str]) -> List[str]:
+    excludes = set(exclude_list)
+    return [col for col in source if col not in excludes]
+
+
+def build_type_hints(cols: List[str]) -> dict:
+    return {col: TYPE_HINTS[col] for col in cols if col in TYPE_HINTS}

ingestr/src/destinations.py

@@ -299,10 +299,21 @@ class ClickhouseDestination:
             )
 
         query_params = parse_qs(parsed_uri.query)
+        secure = int(query_params["secure"][0]) if "secure" in query_params else 1
+
         http_port = (
-            int(query_params["http_port"][0]) if "http_port" in query_params else 8123
+            int(query_params["http_port"][0])
+            if "http_port" in query_params
+            else 8443
+            if secure == 1
+            else 8123
         )
 
+        if secure not in (0, 1):
+            raise ValueError(
+                "Invalid value for secure. Set to `1` for a secure HTTPS connection or `0` for a non-secure HTTP connection."
+            )
+
         credentials = ClickHouseCredentials(
             {
                 "host": host,
@@ -311,10 +322,9 @@ class ClickhouseDestination:
                 "password": password,
                 "database": database,
                 "http_port": http_port,
-                "secure": 0,
+                "secure": secure,
             }
         )
-
         return dlt.destinations.clickhouse(credentials=credentials)
 
     def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:

ingestr/src/factory.py

@@ -20,6 +20,7 @@ from ingestr.src.sources import (
     AdjustSource,
     AirtableSource,
     AppleAppStoreSource,
+    AppLovinSource,
     AppsflyerSource,
     ArrowMemoryMappedSource,
     AsanaSource,
@@ -131,6 +132,7 @@ class SourceDestinationFactory:
         "appstore": AppleAppStoreSource,
         "gs": GCSSource,
         "linkedinads": LinkedInAdsSource,
+        "applovin": AppLovinSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr/src/sources.py

@@ -50,6 +50,7 @@ from ingestr.src import blob
 from ingestr.src.adjust import REQUIRED_CUSTOM_DIMENSIONS, adjust_source
 from ingestr.src.adjust.adjust_helpers import parse_filters
 from ingestr.src.airtable import airtable_source
+from ingestr.src.applovin import applovin_source
 from ingestr.src.appsflyer._init_ import appsflyer_source
 from ingestr.src.appstore import app_store
 from ingestr.src.appstore.client import AppStoreConnectClient
@@ -1737,3 +1738,47 @@ class LinkedInAdsSource:
             metrics=metrics,
             time_granularity=time_granularity,
         ).with_resources("custom_reports")
+
+
+class AppLovinSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key") is not None:
+            raise ValueError(
+                "Google Ads takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+
+        api_key = params.get("api_key", None)
+        if api_key is None:
+            raise MissingValueError("api_key", "AppLovin")
+
+        interval_start = kwargs.get("interval_start")
+        interval_end = kwargs.get("interval_end")
+
+        now = datetime.now()
+        start_date = (
+            interval_start if interval_start is not None else now - timedelta(days=1)
+        )
+        end_date = interval_end
+
+        custom_report = None
+        if table.startswith("custom:"):
+            custom_report = table
+            table = "custom_report"
+
+        src = applovin_source(
+            api_key[0],
+            start_date.strftime("%Y-%m-%d"),
+            end_date.strftime("%Y-%m-%d") if end_date else None,
+            custom_report,
+        )
+
+        if table not in src.resources:
+            raise UnsupportedResourceError(table, "AppLovin")
+
+        return src.with_resources(table)

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.13.1"
+__version__ = "0.13.3"

{ingestr-0.13.1.dist-info → ingestr-0.13.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.1
+Version: 0.13.3
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.1.dist-info → ingestr-0.13.3.dist-info}/RECORD

@@ -1,17 +1,18 @@
 ingestr/main.py,sha256=ufn8AcM2ID80ChUApJzYDjnQaurMXOkYfTm6GzAggSQ,24746
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=XDk_XqmU_He4sQ1brY3ceoZgpq_ZBZihz1gHW9MzqUk,1381
-ingestr/src/destinations.py,sha256=tY9-1mOyGkXl5dbSiE-eOsK-jnbYIu4EaoqhKDZ5JuU,10829
+ingestr/src/destinations.py,sha256=WxerdCx0gS4JveYAE-GzdJLbgP4t2QXXQhUoU3GvmLM,11194
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=3XM2rilA69vkkOCHNzUt1XqCOc3gLMnOnlQmW5d1V5s,4870
+ingestr/src/factory.py,sha256=b1Fg3lhTu6HoALi4AFrTOm3fbNPj5EB2lPigrjiY1so,4926
 ingestr/src/filters.py,sha256=0JQXeAr2APFMnW2sd-6BlAMWv93bXV17j8b5MM8sHmM,580
-ingestr/src/sources.py,sha256=VBuD6ngMHKaCLeYZ9Oe9tw67578hPc1dP_5iBNtEJdM,61683
+ingestr/src/sources.py,sha256=YkBCyjI8DWovyA8CI68zOlHT4UKPsG8-UTCyuafI_WA,63075
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
-ingestr/src/version.py,sha256=Zg3oo58_HXe_ieb_PwWnYkKGH2zTvu6G2jly-7GnPGo,23
+ingestr/src/version.py,sha256=aiCDTKDs80gZjvqiXYkxhq_MLi4Du0L9OByDr6ZEVV4,23
 ingestr/src/adjust/__init__.py,sha256=ULjtJqrNS6XDvUyGl0tjl12-tLyXlCgeFe2icTbtu3Q,3255
 ingestr/src/adjust/adjust_helpers.py,sha256=av97NPSn-hQtTbAC0vUSCAWYePmOiG5R-DGdMssm7FQ,3646
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
+ingestr/src/applovin/__init__.py,sha256=VwVTtVQetnSpUt3cgy6TuH3sYdTnQP63eO_qYqT1TEA,7387
 ingestr/src/appsflyer/_init_.py,sha256=ne2-9FQ654Drtd3GkKQv8Bwb6LEqCnJw49MfO5Jyzgs,739
 ingestr/src/appsflyer/client.py,sha256=TNmwakLzmO6DZW3wcfLfQRl7aNBHgFqSsk4ef-MmJ1w,3084
 ingestr/src/appstore/__init__.py,sha256=3P4VZH2WJF477QjW19jMTwu6L8DXcLkYSdutnvp3AmM,4742
@@ -100,8 +101,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.1.dist-info/METADATA,sha256=c2bUEbUHRYoKiHXNgHSbzNsQL3M2dLiCZK1fzbjzylU,8252
-ingestr-0.13.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.1.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.1.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.1.dist-info/RECORD,,
+ingestr-0.13.3.dist-info/METADATA,sha256=mVT47j7eOqecG3fnDHEkWzqmM4QHWDUixAWKpkGsoEk,8252
+ingestr-0.13.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.3.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.3.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.3.dist-info/RECORD,,