ingestr 0.7.8__py3-none-any.whl → 0.8.2__py3-none-any.whl

ingestr/main.py

@@ -259,6 +259,7 @@ def ingest(
         },
     )
 
+    dlt.config["data_writer.buffer_max_items"] = page_size
     dlt.config["data_writer.file_max_items"] = loader_file_size
     dlt.config["extract.workers"] = extract_parallelism
     dlt.config["extract.max_parallel_items"] = extract_parallelism

ingestr/src/adjust/_init_.py

@@ -0,0 +1,31 @@
+from typing import Sequence
+
+import dlt
+from dlt.sources import DltResource
+
+from .helpers import DEFAULT_DIMENSIONS, AdjustAPI
+
+
+@dlt.source(max_table_nesting=0)
+def adjust_source(
+    start_date: str,
+    end_date: str,
+    api_key: str,
+) -> Sequence[DltResource]:
+    @dlt.resource(write_disposition="merge", merge_key="day")
+    def campaigns():
+        adjust_api = AdjustAPI(api_key=api_key)
+        yield from adjust_api.fetch_report_data(
+            start_date=start_date,
+            end_date=end_date,
+        )
+
+    @dlt.resource(write_disposition="merge", merge_key="day")
+    def creatives():
+        dimensions = DEFAULT_DIMENSIONS + ["adgroup", "creative"]
+        adjust_api = AdjustAPI(api_key=api_key)
+        yield from adjust_api.fetch_report_data(
+            start_date=start_date, end_date=end_date, dimensions=dimensions
+        )
+
+    return campaigns, creatives

ingestr/src/adjust/helpers.py

@@ -0,0 +1,82 @@
+import requests
+from dlt.sources.helpers.requests import Client
+from requests.exceptions import HTTPError
+
+DEFAULT_DIMENSIONS = ["campaign", "day", "app", "store_type", "channel", "country"]
+
+DEFAULT_METRICS = [
+    "network_cost",
+    "all_revenue_total_d0",
+    "ad_revenue_total_d0",
+    "revenue_total_d0",
+    "all_revenue_total_d1",
+    "ad_revenue_total_d1",
+    "revenue_total_d1",
+    "all_revenue_total_d3",
+    "ad_revenue_total_d3",
+    "revenue_total_d3",
+    "all_revenue_total_d7",
+    "ad_revenue_total_d7",
+    "revenue_total_d7",
+    "all_revenue_total_d14",
+    "ad_revenue_total_d14",
+    "revenue_total_d14",
+    "all_revenue_total_d21",
+]
+
+
+class AdjustAPI:
+    def __init__(self, api_key):
+        self.api_key = api_key
+        self.uri = "https://automate.adjust.com/reports-service/report"
+
+    def fetch_report_data(
+        self,
+        start_date,
+        end_date,
+        dimensions=DEFAULT_DIMENSIONS,
+        metrics=DEFAULT_METRICS,
+        utc_offset="+00:00",
+        ad_spend_mode="network",
+        attribution_source="first",
+        attribution_type="all",
+        cohort_maturity="immature",
+        reattributed="all",
+        sandbox="false",
+    ):
+        headers = {"Authorization": f"Bearer {self.api_key}"}
+        comma_separated_dimensions = ",".join(dimensions)
+        comma_separated_metrics = ",".join(metrics)
+        params = {
+            "date_period": f"{start_date}:{end_date}",
+            "dimensions": comma_separated_dimensions,
+            "metrics": comma_separated_metrics,
+            "utc_offset": utc_offset,
+            "ad_spend_mode": ad_spend_mode,
+            "attribution_source": attribution_source,
+            "attribution_type": attribution_type,
+            "cohort_maturity": cohort_maturity,
+            "reattributed": reattributed,
+            "sandbox": sandbox,
+        }
+
+        def retry_on_limit(
+            response: requests.Response, exception: BaseException
+        ) -> bool:
+            return response.status_code == 429
+
+        request_client = Client(
+            request_timeout=8.0,
+            raise_for_status=False,
+            retry_condition=retry_on_limit,
+            request_max_attempts=12,
+            request_backoff_factor=2,
+        ).session
+
+        response = request_client.get(self.uri, headers=headers, params=params)
+        if response.status_code == 200:
+            result = response.json()
+            items = result.get("rows", [])
+            yield items
+        else:
+            raise HTTPError(f"Request failed with status code: {response.status_code}")

ingestr/src/appsflyer/_init_.py

@@ -0,0 +1,24 @@
+from typing import Iterable
+
+import dlt
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from ingestr.src.appsflyer.client import AppsflyerClient
+
+
+@dlt.source(max_table_nesting=0)
+def appsflyer_source(
+    api_key: str, start_date: str, end_date: str
+) -> Iterable[DltResource]:
+    client = AppsflyerClient(api_key)
+
+    @dlt.resource(write_disposition="merge", merge_key="install_time")
+    def campaigns() -> Iterable[TDataItem]:
+        yield from client.fetch_campaigns(start_date, end_date)
+
+    @dlt.resource(write_disposition="merge", merge_key="install_time")
+    def creatives() -> Iterable[TDataItem]:
+        yield from client.fetch_creatives(start_date, end_date)
+
+    return campaigns, creatives

ingestr/src/appsflyer/client.py

@@ -0,0 +1,106 @@
+from typing import Optional
+
+import requests
+from dlt.sources.helpers.requests import Client
+from requests.exceptions import HTTPError
+
+DEFAULT_GROUPING = ["c", "geo", "app_id", "install_time"]
+DEFAULT_KPIS = [
+    "impressions",
+    "clicks",
+    "installs",
+    "cost",
+    "revenue",
+    "average_ecpi",
+    "loyal_users",
+    "uninstalls",
+    "roi",
+]
+
+
+class AppsflyerClient:
+    def __init__(self, api_key: str):
+        self.api_key = api_key
+        self.uri = "https://hq1.appsflyer.com/api/master-agg-data/v4/app/all"
+
+    def __get_headers(self):
+        return {
+            "Authorization": f"{self.api_key}",
+            "accept": "text/json",
+        }
+
+    def _fetch_data(
+        self,
+        from_date: str,
+        to_date: str,
+        maximum_rows=1000000,
+        dimensions=DEFAULT_GROUPING,
+        metrics=DEFAULT_KPIS,
+    ):
+        params = {
+            "from": from_date,
+            "to": to_date,
+            "groupings": ",".join(dimensions),
+            "kpis": ",".join(metrics),
+            "format": "json",
+            "maximum_rows": maximum_rows,
+        }
+
+        def retry_on_limit(
+            response: Optional[requests.Response], exception: Optional[BaseException]
+        ) -> bool:
+            return (
+                isinstance(response, requests.Response) and response.status_code == 429
+            )
+
+        request_client = Client(
+            request_timeout=10.0,
+            raise_for_status=False,
+            retry_condition=retry_on_limit,
+            request_max_attempts=12,
+            request_backoff_factor=2,
+        ).session
+
+        try:
+            response = request_client.get(
+                url=self.uri, headers=self.__get_headers(), params=params
+            )
+
+            if response.status_code == 200:
+                result = response.json()
+                yield result
+            else:
+                raise HTTPError(
+                    f"Request failed with status code: {response.status_code}"
+                )
+
+        except requests.RequestException as e:
+            raise HTTPError(f"Request failed: {e}")
+
+    def fetch_campaigns(
+        self,
+        start_date: str,
+        end_date: str,
+    ):
+        metrics = DEFAULT_KPIS + [
+            "cohort_day_1_revenue_per_user",
+            "cohort_day_1_total_revenue_per_user",
+            "cohort_day_3_revenue_per_user",
+            "cohort_day_3_total_revenue_per_user",
+            "cohort_day_7_total_revenue_per_user",
+            "cohort_day_7_revenue_per_user",
+            "cohort_day_14_total_revenue_per_user",
+            "cohort_day_14_revenue_per_user",
+            "cohort_day_21_total_revenue_per_user",
+            "cohort_day_21_revenue_per_user",
+            "retention_day_7",
+        ]
+        return self._fetch_data(start_date, end_date, metrics=metrics)
+
+    def fetch_creatives(
+        self,
+        start_date: str,
+        end_date: str,
+    ):
+        dimensions = DEFAULT_GROUPING + ["af_adset_id", "af_adset", "af_ad_id"]
+        return self._fetch_data(start_date, end_date, dimensions=dimensions)

ingestr/src/factory.py

@@ -15,7 +15,9 @@ from ingestr.src.destinations import (
     SynapseDestination,
 )
 from ingestr.src.sources import (
+    AdjustSource,
     AirtableSource,
+    AppsflyerSource,
     ChessSource,
     FacebookAdsSource,
     GoogleSheetsSource,
@@ -124,8 +126,12 @@ class SourceDestinationFactory:
             return AirtableSource()
         elif self.source_scheme == "klaviyo":
             return KlaviyoSource()
+        elif self.source_scheme == "appsflyer":
+            return AppsflyerSource()
         elif self.source_scheme == "kafka":
             return KafkaSource()
+        elif self.source_scheme == "adjust":
+            return AdjustSource()
         else:
             raise ValueError(f"Unsupported source scheme: {self.source_scheme}")