ingestr 0.13.4__py3-none-any.whl → 0.13.6__py3-none-any.whl

ingestr/src/applovin/__init__.py

@@ -1,6 +1,7 @@
-from datetime import datetime, timezone
+from datetime import datetime, timezone, timedelta
 from enum import Enum
 from typing import Dict, List, Optional
+from requests import Response
 
 import dlt
 from dlt.sources.rest_api import EndpointResource, RESTAPIConfig, rest_api_resources
@@ -12,11 +13,8 @@ class InvalidCustomReportError(Exception):
             "Custom report should be in the format 'custom:{endpoint}:{report_type}:{dimensions}"
         )
 
-
-class InvalidDimensionError(Exception):
-    def __init__(self, dim: str, report_type: str):
-        super().__init__(f"Unknown dimension {dim} for report type {report_type}")
-
+class ClientError(Exception):
+    pass
 
 TYPE_HINTS = {
     "application_is_hidden": {"data_type": "bool"},
@@ -108,32 +106,6 @@ REPORT_SCHEMA: Dict[ReportType, List[str]] = {
     ],
 }
 
-# NOTE(turtledev): These values are valid columns,
-# but often don't produce a value. Find a way to either add
-# a default value, or use an alternative strategy to de-duplicate
-# OR make them nullable
-SKA_REPORT_EXCLUDE = [
-    "ad",
-    "ad_id",
-    "ad_type",
-    "average_cpc",
-    "campaign_ad_type",
-    "clicks",
-    "conversions",
-    "conversion_rate",
-    "creative_set",
-    "creative_set_id",
-    "ctr",
-    "custom_page_id",
-    "device_type",
-    "first_purchase",
-    "impressions",
-    "placement_type",
-    "sales",
-    "size",
-    "traffic_source",
-]
-
 PROBABILISTIC_REPORT_EXCLUDE = [
     "installs",
     "redownloads",
@@ -147,19 +119,16 @@ def applovin_source(
     end_date: Optional[str],
     custom: Optional[str],
 ):
-    ska_report_columns = exclude(
-        REPORT_SCHEMA[ReportType.ADVERTISER],
-        SKA_REPORT_EXCLUDE,
-    )
 
-    probabilistic_report_columns = exclude(
-        REPORT_SCHEMA[ReportType.ADVERTISER],
-        PROBABILISTIC_REPORT_EXCLUDE,
-    )
     backfill = False
     if end_date is None:
         backfill = True
-        end_date = datetime.now(timezone.utc).date().strftime("%Y-%m-%d")
+
+        # use the greatest of yesterday and start_date
+        end_date = max(
+            datetime.now(timezone.utc) - timedelta(days=1),
+            datetime.fromisoformat(start_date).replace(tzinfo=timezone.utc)
+        ).strftime("%Y-%m-%d")
 
     config: RESTAPIConfig = {
         "client": {
@@ -186,6 +155,9 @@ def applovin_source(
                     "end": end_date,
                 },
                 "paginator": "single_page",
+                "response_actions": [
+                    http_error_handler,
+                ]
             },
         },
         "resources": [
@@ -204,13 +176,16 @@ def applovin_source(
             resource(
                 "advertiser-probabilistic-report",
                 "probabilisticReport",
-                probabilistic_report_columns,
+                exclude(
+                    REPORT_SCHEMA[ReportType.ADVERTISER],
+                    PROBABILISTIC_REPORT_EXCLUDE
+                ),
                 ReportType.ADVERTISER,
             ),
             resource(
                 "advertiser-ska-report",
                 "skaReport",
-                ska_report_columns,
+                REPORT_SCHEMA[ReportType.ADVERTISER],
                 ReportType.ADVERTISER,
             ),
         ],
@@ -280,3 +255,7 @@ def exclude(source: List[str], exclude_list: List[str]) -> List[str]:
 
 def build_type_hints(cols: List[str]) -> dict:
     return {col: TYPE_HINTS[col] for col in cols if col in TYPE_HINTS}
+
+def http_error_handler(resp: Response):
+    if not resp.ok:
+        raise ClientError(f"HTTP Status {resp.status_code}: {resp.text}")

ingestr/src/applovin_max/__init__.py

@@ -0,0 +1,99 @@
+from typing import Iterator
+
+import dlt
+import pandas as pd  # type: ignore[import-untyped]
+import pendulum
+import requests
+from dlt.sources import DltResource
+from dlt.sources.helpers.requests import Client
+from pendulum.date import Date
+
+
+@dlt.source(max_table_nesting=0)
+def applovin_max_source(
+    start_date: str,
+    application: str,
+    api_key: str,
+    end_date: str | None,
+) -> DltResource:
+    @dlt.resource(
+        name="ad_revenue",
+        write_disposition="merge",
+        merge_key="_partition_date",
+    )
+    def fetch_ad_revenue_report(
+        dateTime=(
+            dlt.sources.incremental(
+                "_partition_date",
+                initial_value=start_date,
+                end_value=end_date,
+                range_start="closed",
+                range_end="closed",
+            )
+        ),
+    ) -> Iterator[dict]:
+        url = "https://r.applovin.com/max/userAdRevenueReport"
+        start_date = pendulum.from_format(dateTime.last_value, "YYYY-MM-DD").date()
+        if dateTime.end_value is None:
+            end_date = (pendulum.yesterday("UTC")).date()
+        else:
+            end_date = pendulum.from_format(dateTime.end_value, "YYYY-MM-DD").date()
+        yield get_data(
+            url=url,
+            start_date=start_date,
+            end_date=end_date,
+            application=application,
+            api_key=api_key,
+        )
+
+    return fetch_ad_revenue_report
+
+
+def create_client() -> requests.Session:
+    return Client(
+        request_timeout=10.0,
+        raise_for_status=False,
+        retry_condition=retry_on_limit,
+        request_max_attempts=12,
+    ).session
+
+
+def retry_on_limit(
+    response: requests.Response | None, exception: BaseException | None
+) -> bool:
+    if response is None:
+        return False
+    return response.status_code == 429
+
+
+def get_data(
+    url: str, start_date: Date, end_date: Date, application: str, api_key: str
+):
+    client = create_client()
+    platforms = ["ios", "android", "fireos"]
+    current_date = start_date
+    while current_date <= end_date:
+        for platform in platforms:
+            params = {
+                "api_key": api_key,
+                "date": current_date.strftime("%Y-%m-%d"),
+                "platform": platform,
+                "application": application,
+                "aggregated": "false",
+            }
+
+            response = client.get(url=url, params=params)
+
+            if response.status_code == 400:
+                raise ValueError(response.text)
+
+            if response.status_code != 200:
+                continue
+
+            response_url = response.json().get("ad_revenue_report_url")
+            df = pd.read_csv(response_url)
+            df["Date"] = pd.to_datetime(df["Date"])
+            df["_partition_date"] = df["Date"].dt.strftime("%Y-%m-%d")
+            yield df
+
+        current_date = current_date.add(days=1)

ingestr/src/destinations.py

@@ -60,9 +60,14 @@ class BigQueryDestination:
                 base64.b64decode(credentials_base64[0]).decode("utf-8")
             )
 
+        project_id = None
+        if source_fields.hostname:
+            project_id = source_fields.hostname
+
         return dlt.destinations.bigquery(
             credentials=credentials,  # type: ignore
             location=location,
+            project_id=project_id,
             **kwargs,
         )
 

ingestr/src/factory.py

@@ -20,6 +20,7 @@ from ingestr.src.sources import (
     AdjustSource,
     AirtableSource,
     AppleAppStoreSource,
+    ApplovinMaxSource,
     AppLovinSource,
     AppsflyerSource,
     ArrowMemoryMappedSource,
@@ -67,9 +68,9 @@ SQL_SOURCE_SCHEMES = [
     "oracle+cx_oracle",
     "hana",
     "clickhouse",
-    
 ]
 
+
 class SourceProtocol(Protocol):
     def dlt_source(self, uri: str, table: str, **kwargs):
         pass
@@ -134,6 +135,7 @@ class SourceDestinationFactory:
         "gs": GCSSource,
         "linkedinads": LinkedInAdsSource,
         "applovin": AppLovinSource,
+        "applovinmax": ApplovinMaxSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr/src/sources.py

@@ -51,6 +51,7 @@ from ingestr.src.adjust import REQUIRED_CUSTOM_DIMENSIONS, adjust_source
 from ingestr.src.adjust.adjust_helpers import parse_filters
 from ingestr.src.airtable import airtable_source
 from ingestr.src.applovin import applovin_source
+from ingestr.src.applovin_max import applovin_max_source
 from ingestr.src.appsflyer._init_ import appsflyer_source
 from ingestr.src.appstore import app_store
 from ingestr.src.appstore.client import AppStoreConnectClient
@@ -132,11 +133,11 @@ class SqlSource:
 
         if uri.startswith("mysql://"):
             uri = uri.replace("mysql://", "mysql+pymysql://")
-        
+
         if uri.startswith("clickhouse://"):
             uri = uri.replace("clickhouse://", "clickhouse+native://")
             if "secure=" not in uri:
-              uri += "?secure=1"
+                uri += "?secure=1"
 
         query_adapters = []
         if kwargs.get("sql_limit"):
@@ -1787,3 +1788,48 @@ class AppLovinSource:
             raise UnsupportedResourceError(table, "AppLovin")
 
         return src.with_resources(table)
+
+
+class ApplovinMaxSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+
+        api_key = params.get("api_key")
+        if api_key is None:
+            raise ValueError("api_key is required to connect to AppLovin Max API.")
+
+        application = params.get("application")
+        if application is None:
+            raise ValueError("application is required to connect to AppLovin Max API.")
+
+        interval_start = kwargs.get("interval_start")
+        interval_end = kwargs.get("interval_end")
+
+        if "ad_revenue" in table:
+            table = "ad_revenue"
+        else:
+            raise ValueError(
+                f"Table name '{table}' is not supported for AppLovin Max source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+            )
+
+        now = pendulum.now("UTC")
+        default_start = now.subtract(days=30).date()
+
+        start_date = (
+            interval_start if interval_start is not None else default_start
+        ).strftime("%Y-%m-%d")
+
+        end_date = (
+            interval_end.strftime("%Y-%m-%d") if interval_end is not None else None
+        )
+
+        return applovin_max_source(
+            start_date=start_date,
+            end_date=end_date,
+            api_key=api_key[0],
+            application=application[0],
+        ).with_resources(table)

{ingestr-0.13.4.dist-info → ingestr-0.13.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.4
+Version: 0.13.6
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.4.dist-info → ingestr-0.13.6.dist-info}/RECORD

@@ -1,19 +1,20 @@
 ingestr/main.py,sha256=ufn8AcM2ID80ChUApJzYDjnQaurMXOkYfTm6GzAggSQ,24746
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=XDk_XqmU_He4sQ1brY3ceoZgpq_ZBZihz1gHW9MzqUk,1381
-ingestr/src/destinations.py,sha256=v45jYRVGXCof1XEvl07XlkVkxeIMiGmBwTrZT8-rQAI,11134
+ingestr/src/destinations.py,sha256=vrGij4qMPCdXTMIimROWBJFqzOqCM4DFmgyubgSHejA,11279
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=_VkpHRKCLP7UGm2-iHbXZOHuuvfMduoXo_oNZSX-mSw,4948
+ingestr/src/factory.py,sha256=XYwjy5dfG5mLIU1v-mS17Kwl0cxSs3MG7NtgPPwZ_0U,5009
 ingestr/src/filters.py,sha256=0JQXeAr2APFMnW2sd-6BlAMWv93bXV17j8b5MM8sHmM,580
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
-ingestr/src/sources.py,sha256=PkAjowbUF1LxaLshn2x39Mi5lzIOVm9speX7gTMTFzE,63269
+ingestr/src/sources.py,sha256=ljh__y_ZXj8NUT0v63ZAT42K1SZsEJEB88YtQHG0IXQ,64830
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=l6zVm0GMMwnBlIOONWc6snhko9d8-HO1y6Jj1T1vsiQ,158
 ingestr/src/adjust/__init__.py,sha256=ULjtJqrNS6XDvUyGl0tjl12-tLyXlCgeFe2icTbtu3Q,3255
 ingestr/src/adjust/adjust_helpers.py,sha256=av97NPSn-hQtTbAC0vUSCAWYePmOiG5R-DGdMssm7FQ,3646
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
-ingestr/src/applovin/__init__.py,sha256=VwVTtVQetnSpUt3cgy6TuH3sYdTnQP63eO_qYqT1TEA,7387
+ingestr/src/applovin/__init__.py,sha256=vtmYnRKnNOSzFWQIbKGbrcu6AcBdHuhPMsNruUvEIgg,7000
+ingestr/src/applovin_max/__init__.py,sha256=1NUOeJzRyZZQ95KEirbrlSrk-8SNc9JrlM_5pGgBgHg,2878
 ingestr/src/appsflyer/_init_.py,sha256=ne2-9FQ654Drtd3GkKQv8Bwb6LEqCnJw49MfO5Jyzgs,739
 ingestr/src/appsflyer/client.py,sha256=TNmwakLzmO6DZW3wcfLfQRl7aNBHgFqSsk4ef-MmJ1w,3084
 ingestr/src/appstore/__init__.py,sha256=3P4VZH2WJF477QjW19jMTwu6L8DXcLkYSdutnvp3AmM,4742
@@ -102,8 +103,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.4.dist-info/METADATA,sha256=SVXv8u5Rz4HUCIx_sYfBwff4MmtSp7UyqPCU17O4Krc,8956
-ingestr-0.13.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.4.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.4.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.4.dist-info/RECORD,,
+ingestr-0.13.6.dist-info/METADATA,sha256=ylIPPjjCbE-qTxxYtx3OXydv-uOYuQvD-qNc13fs-Bk,8956
+ingestr-0.13.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.6.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.6.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.6.dist-info/RECORD,,