omniload 0.0.0.dev0__py3-none-any.whl

omniload/src/appsflyer/__init__.py

@@ -0,0 +1,325 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from omniload.src.appsflyer.client import AppsflyerClient
+
+DIMENSION_RESPONSE_MAPPING = {
+    "c": "campaign",
+    "af_adset_id": "adset_id",
+    "af_adset": "adset",
+    "af_ad_id": "ad_id",
+}
+HINTS = {
+    "app_id": {
+        "data_type": "text",
+        "nullable": False,
+    },
+    "campaign": {
+        "data_type": "text",
+        "nullable": False,
+    },
+    "geo": {
+        "data_type": "text",
+        "nullable": False,
+    },
+    "cost": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": False,
+    },
+    "clicks": {
+        "data_type": "bigint",
+        "nullable": False,
+    },
+    "impressions": {
+        "data_type": "bigint",
+        "nullable": False,
+    },
+    "average_ecpi": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": False,
+    },
+    "installs": {
+        "data_type": "bigint",
+        "nullable": False,
+    },
+    "retention_day_7": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": False,
+    },
+    "retention_day_14": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": False,
+    },
+    "cohort_day_1_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_1_total_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_3_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_3_total_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_7_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_7_total_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_14_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_14_total_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_21_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_21_total_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "install_time": {
+        "data_type": "date",
+        "nullable": False,
+    },
+    "loyal_users": {
+        "data_type": "bigint",
+        "nullable": False,
+    },
+    "revenue": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "roi": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "uninstalls": {
+        "data_type": "bigint",
+        "nullable": True,
+    },
+}
+
+CAMPAIGNS_DIMENSIONS = ["c", "geo", "app_id", "install_time"]
+CAMPAIGNS_METRICS = [
+    "average_ecpi",
+    "clicks",
+    "cohort_day_1_revenue_per_user",
+    "cohort_day_1_total_revenue_per_user",
+    "cohort_day_14_revenue_per_user",
+    "cohort_day_14_total_revenue_per_user",
+    "cohort_day_21_revenue_per_user",
+    "cohort_day_21_total_revenue_per_user",
+    "cohort_day_3_revenue_per_user",
+    "cohort_day_3_total_revenue_per_user",
+    "cohort_day_7_revenue_per_user",
+    "cohort_day_7_total_revenue_per_user",
+    "cost",
+    "impressions",
+    "installs",
+    "loyal_users",
+    "retention_day_7",
+    "revenue",
+    "roi",
+    "uninstalls",
+]
+
+CREATIVES_DIMENSIONS = [
+    "c",
+    "geo",
+    "app_id",
+    "install_time",
+    "af_adset_id",
+    "af_adset",
+    "af_ad_id",
+]
+CREATIVES_METRICS = [
+    "impressions",
+    "clicks",
+    "installs",
+    "cost",
+    "revenue",
+    "average_ecpi",
+    "loyal_users",
+    "uninstalls",
+    "roi",
+]
+
+
+@dlt.source(max_table_nesting=0)
+def appsflyer_source(
+    api_key: str,
+    start_date: str,
+    end_date: str,
+    dimensions: list[str],
+    metrics: list[str],
+) -> Iterable[DltResource]:
+    client = AppsflyerClient(api_key)
+
+    @dlt.resource(
+        write_disposition="merge",
+        merge_key="install_time",
+        columns=make_hints(CAMPAIGNS_DIMENSIONS, CAMPAIGNS_METRICS),
+    )
+    def campaigns(
+        datetime=dlt.sources.incremental(
+            "install_time",
+            initial_value=(
+                start_date
+                if start_date
+                else pendulum.today().subtract(days=30).format("YYYY-MM-DD")
+            ),
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        end = (
+            datetime.end_value
+            if datetime.end_value
+            else pendulum.now().format("YYYY-MM-DD")
+        )
+
+        yield from client._fetch_data(
+            from_date=datetime.last_value,
+            to_date=end,
+            dimensions=CAMPAIGNS_DIMENSIONS,
+            metrics=CAMPAIGNS_METRICS,
+        )
+
+    @dlt.resource(
+        write_disposition="merge",
+        merge_key="install_time",
+        columns=make_hints(CREATIVES_DIMENSIONS, CREATIVES_METRICS),
+    )
+    def creatives(
+        datetime=dlt.sources.incremental(
+            "install_time",
+            initial_value=(
+                start_date
+                if start_date
+                else pendulum.today().subtract(days=30).format("YYYY-MM-DD")
+            ),
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        end = (
+            datetime.end_value
+            if datetime.end_value
+            else pendulum.now().format("YYYY-MM-DD")
+        )
+        yield from client._fetch_data(
+            datetime.last_value,
+            end,
+            dimensions=CREATIVES_DIMENSIONS,
+            metrics=CREATIVES_METRICS,
+        )
+
+    primary_keys = []
+    if "install_time" not in dimensions:
+        dimensions.append("install_time")
+        primary_keys.append("install_time")
+
+    for dimension in dimensions:
+        if dimension in DIMENSION_RESPONSE_MAPPING:
+            primary_keys.append(DIMENSION_RESPONSE_MAPPING[dimension])
+        else:
+            primary_keys.append(dimension)
+
+    @dlt.resource(
+        write_disposition="merge",
+        primary_key=primary_keys,
+        columns=make_hints(dimensions, metrics),
+    )
+    def custom(
+        datetime=dlt.sources.incremental(
+            "install_time",
+            initial_value=(
+                start_date
+                if start_date
+                else pendulum.today().subtract(days=30).format("YYYY-MM-DD")
+            ),
+            end_value=end_date,
+        ),
+    ):
+        end = (
+            datetime.end_value
+            if datetime.end_value
+            else pendulum.now().format("YYYY-MM-DD")
+        )
+        res = client._fetch_data(
+            from_date=datetime.last_value,
+            to_date=end,
+            dimensions=dimensions,
+            metrics=metrics,
+        )
+        yield from res
+
+    return campaigns, creatives, custom
+
+
+def make_hints(dimensions: list[str], metrics: list[str]):
+    campaign_hints = {}
+    for dimension in dimensions:
+        resp_key = dimension
+        if dimension in DIMENSION_RESPONSE_MAPPING:
+            resp_key = DIMENSION_RESPONSE_MAPPING[dimension]
+
+        if resp_key in HINTS:
+            campaign_hints[resp_key] = HINTS[resp_key]
+
+    for metric in metrics:
+        if metric in HINTS:
+            campaign_hints[metric] = HINTS[metric]
+
+    return campaign_hints

omniload/src/appsflyer/client.py

@@ -0,0 +1,110 @@
+from typing import Optional
+
+import requests
+from dlt.sources.helpers.requests import Client
+from requests.exceptions import HTTPError
+
+
+class AppsflyerClient:
+    def __init__(self, api_key: str):
+        self.api_key = api_key
+        self.uri = "https://hq1.appsflyer.com/api/master-agg-data/v4/app/all"
+
+    def __get_headers(self):
+        return {
+            "Authorization": f"{self.api_key}",
+            "accept": "text/json",
+        }
+
+    def _fetch_data(
+        self,
+        from_date: str,
+        to_date: str,
+        dimensions: list[str],
+        metrics: list[str],
+        maximum_rows=1000000,
+    ):
+        excluded_metrics = exclude_metrics_for_date_range(metrics, from_date, to_date)
+        included_metrics = [
+            metric for metric in metrics if metric not in excluded_metrics
+        ]
+
+        params = {
+            "from": from_date,
+            "to": to_date,
+            "groupings": ",".join(dimensions),
+            "kpis": ",".join(included_metrics),
+            "format": "json",
+            "maximum_rows": maximum_rows,
+        }
+
+        def retry_on_limit(
+            response: Optional[requests.Response], exception: Optional[BaseException]
+        ) -> bool:
+            return (
+                isinstance(response, requests.Response) and response.status_code == 429
+            )
+
+        request_client = Client(
+            raise_for_status=False,
+            retry_condition=retry_on_limit,
+            request_max_attempts=12,
+            request_backoff_factor=2,
+        ).session
+
+        try:
+            response = request_client.get(
+                url=self.uri, headers=self.__get_headers(), params=params
+            )
+
+            if response.status_code == 200:
+                result = response.json()
+                yield standardize_keys(result, excluded_metrics)
+            else:
+                raise HTTPError(
+                    f"Request failed with status code: {response.status_code}: {response.text}"
+                )
+
+        except requests.RequestException as e:
+            raise HTTPError(f"Request failed: {e}")
+
+
+def standardize_keys(data: list[dict], excluded_metrics: list[str]) -> list[dict]:
+    def fix_key(key: str) -> str:
+        return key.lower().replace("-", "").replace("  ", "_").replace(" ", "_")
+
+    standardized = []
+    for item in data:
+        standardized_item = {}
+        for key, value in item.items():
+            standardized_item[fix_key(key)] = value
+
+        for metric in excluded_metrics:
+            if metric not in standardized_item:
+                standardized_item[fix_key(metric)] = None
+
+        standardized.append(standardized_item)
+
+    return standardized
+
+
+def exclude_metrics_for_date_range(
+    metrics: list[str], from_date: str, to_date: str
+) -> list[str]:
+    """
+    Some of the cohort metrics are not available if there hasn't been enough time to have data for that cohort.
+    This means if you request data for yesterday with cohort day 7 metrics, you will get an error because 7 days hasn't passed yet.
+    One would expect the API to handle this gracefully, but it doesn't.
+
+    This function will exclude the metrics that are not available for the given date range.
+    """
+    import pendulum
+
+    excluded_metrics = []
+    days_between_today_and_end = (pendulum.now() - pendulum.parse(to_date)).days  # type: ignore
+    for metric in metrics:
+        if "cohort_day_" in metric:
+            day_count = int(metric.split("_")[2])
+            if days_between_today_and_end <= day_count:
+                excluded_metrics.append(metric)
+    return excluded_metrics

omniload/src/appstore/__init__.py

@@ -0,0 +1,142 @@
+import csv
+import gzip
+import os
+import tempfile
+from copy import deepcopy
+from datetime import datetime
+from typing import Iterable, List, Optional
+
+import dlt
+import requests
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from .client import AppStoreConnectClientInterface
+from .errors import (
+    NoOngoingReportRequestsFoundError,
+    NoReportsFoundError,
+    NoSuchReportError,
+)
+from .models import AnalyticsReportInstancesResponse
+from .resources import RESOURCES
+
+
+@dlt.source
+def app_store(
+    client: AppStoreConnectClientInterface,
+    app_ids: List[str],
+    start_date: Optional[datetime] = None,
+    end_date: Optional[datetime] = None,
+) -> Iterable[DltResource]:
+    if start_date and start_date.tzinfo is not None:
+        start_date = start_date.replace(tzinfo=None)
+    if end_date and end_date.tzinfo is not None:
+        end_date = end_date.replace(tzinfo=None)
+    for resource in RESOURCES:
+        yield dlt.resource(
+            get_analytics_reports,
+            name=resource.name,
+            primary_key=resource.primary_key,
+            columns=resource.columns,
+            write_disposition="merge",
+        )(client, app_ids, resource.report_name, start_date, end_date)
+
+
+def filter_instances_by_date(
+    instances: AnalyticsReportInstancesResponse,
+    start_date: Optional[datetime],
+    end_date: Optional[datetime],
+) -> AnalyticsReportInstancesResponse:
+    instances = deepcopy(instances)
+    if start_date is not None:
+        instances.data = list(
+            filter(
+                lambda x: datetime.fromisoformat(x.attributes.processingDate)
+                >= start_date,
+                instances.data,
+            )
+        )
+    if end_date is not None:
+        instances.data = list(
+            filter(
+                lambda x: datetime.fromisoformat(x.attributes.processingDate)
+                <= end_date,
+                instances.data,
+            )
+        )
+
+    return instances
+
+
+def get_analytics_reports(
+    client: AppStoreConnectClientInterface,
+    app_ids: List[str],
+    report_name: str,
+    start_date: Optional[datetime],
+    end_date: Optional[datetime],
+    last_processing_date=dlt.sources.incremental("processing_date"),
+) -> Iterable[TDataItem]:
+    if last_processing_date.last_value:
+        start_date = datetime.fromisoformat(last_processing_date.last_value)
+    for app_id in app_ids:
+        yield from get_report(client, app_id, report_name, start_date, end_date)
+
+
+def get_report(
+    client: AppStoreConnectClientInterface,
+    app_id: str,
+    report_name: str,
+    start_date: Optional[datetime],
+    end_date: Optional[datetime],
+) -> Iterable[TDataItem]:
+    report_requests = client.list_analytics_report_requests(app_id)
+    ongoing_requests = list(
+        filter(
+            lambda x: x.attributes.accessType == "ONGOING"
+            and not x.attributes.stoppedDueToInactivity,
+            report_requests.data,
+        )
+    )
+
+    if len(ongoing_requests) == 0:
+        raise NoOngoingReportRequestsFoundError()
+
+    reports = client.list_analytics_reports(ongoing_requests[0].id, report_name)
+    if len(reports.data) == 0:
+        raise NoSuchReportError(report_name)
+
+    for report in reports.data:
+        instances = client.list_report_instances(report.id)
+
+        instances = filter_instances_by_date(instances, start_date, end_date)
+
+        if len(instances.data) == 0:
+            raise NoReportsFoundError()
+
+        for instance in instances.data:
+            segments = client.list_report_segments(instance.id)
+            with tempfile.TemporaryDirectory() as temp_dir:
+                files = []
+                for segment in segments.data:
+                    payload = requests.get(segment.attributes.url, stream=True)
+                    payload.raise_for_status()
+
+                    csv_path = os.path.join(
+                        temp_dir, f"{segment.attributes.checksum}.csv"
+                    )
+                    with open(csv_path, "wb") as f:
+                        for chunk in payload.iter_content(chunk_size=8192):
+                            f.write(chunk)
+                    files.append(csv_path)
+                for file in files:
+                    with gzip.open(file, "rt") as f:
+                        # TODO: infer delimiter from the file itself
+                        delimiter = (
+                            "," if report_name == "App Crashes Expanded" else "\t"
+                        )
+                        reader = csv.DictReader(f, delimiter=delimiter)
+                        for row in reader:
+                            yield {
+                                "processing_date": instance.attributes.processingDate,
+                                **row,
+                            }

omniload/src/appstore/client.py

@@ -0,0 +1,126 @@
+import abc
+import time
+from typing import Optional
+
+import jwt
+import requests
+from requests.models import PreparedRequest
+
+from .models import (
+    AnalyticsReportInstancesResponse,
+    AnalyticsReportRequestsResponse,
+    AnalyticsReportResponse,
+    AnalyticsReportSegmentsResponse,
+)
+
+
+class AppStoreConnectClientInterface(abc.ABC):
+    @abc.abstractmethod
+    def list_analytics_report_requests(self, app_id) -> AnalyticsReportRequestsResponse:
+        pass
+
+    @abc.abstractmethod
+    def list_analytics_reports(
+        self, req_id: str, report_name: str
+    ) -> AnalyticsReportResponse:
+        pass
+
+    @abc.abstractmethod
+    def list_report_instances(
+        self,
+        report_id: str,
+        granularity: str = "DAILY",
+    ) -> AnalyticsReportInstancesResponse:
+        pass
+
+    @abc.abstractmethod
+    def list_report_segments(self, instance_id: str) -> AnalyticsReportSegmentsResponse:
+        pass
+
+
+class AppStoreConnectClient(AppStoreConnectClientInterface):
+    def __init__(self, key: bytes, key_id: str, issuer_id: str):
+        self.__key = key
+        self.__key_id = key_id
+        self.__issuer_id = issuer_id
+
+    def list_analytics_report_requests(self, app_id) -> AnalyticsReportRequestsResponse:
+        res = requests.get(
+            f"https://api.appstoreconnect.apple.com/v1/apps/{app_id}/analyticsReportRequests",
+            auth=self.auth,
+        )
+        res.raise_for_status()
+
+        return AnalyticsReportRequestsResponse.from_json(res.text)  # type: ignore
+
+    def list_analytics_reports(
+        self, req_id: str, report_name: str
+    ) -> AnalyticsReportResponse:
+        params = {"filter[name]": report_name}
+        res = requests.get(
+            f"https://api.appstoreconnect.apple.com/v1/analyticsReportRequests/{req_id}/reports",
+            auth=self.auth,
+            params=params,
+        )
+        res.raise_for_status()
+        return AnalyticsReportResponse.from_json(res.text)  # type: ignore
+
+    def list_report_instances(
+        self,
+        report_id: str,
+        granularity: str = "DAILY",
+    ) -> AnalyticsReportInstancesResponse:
+        data = []
+        url = f"https://api.appstoreconnect.apple.com/v1/analyticsReports/{report_id}/instances"
+        params: Optional[dict] = {"filter[granularity]": granularity}
+
+        while url:
+            res = requests.get(url, auth=self.auth, params=params)
+            res.raise_for_status()
+
+            response_data = AnalyticsReportInstancesResponse.from_json(res.text)  # type: ignore
+            data.extend(response_data.data)
+
+            url = response_data.links.next
+            params = None  # Clear params for subsequent requests
+
+        return AnalyticsReportInstancesResponse(
+            data=data,
+            links=response_data.links,
+            meta=response_data.meta,
+        )
+
+    def list_report_segments(self, instance_id: str) -> AnalyticsReportSegmentsResponse:
+        segments = []
+        url = f"https://api.appstoreconnect.apple.com/v1/analyticsReportInstances/{instance_id}/segments"
+
+        while url:
+            res = requests.get(url, auth=self.auth)
+            res.raise_for_status()
+
+            response_data = AnalyticsReportSegmentsResponse.from_json(res.text)  # type: ignore
+            segments.extend(response_data.data)
+
+            url = response_data.links.next
+
+        return AnalyticsReportSegmentsResponse(
+            data=segments, links=response_data.links, meta=response_data.meta
+        )
+
+    def auth(self, req: PreparedRequest) -> PreparedRequest:
+        headers = {
+            "alg": "ES256",
+            "kid": self.__key_id,
+        }
+        payload = {
+            "iss": self.__issuer_id,
+            "exp": int(time.time()) + 600,
+            "aud": "appstoreconnect-v1",
+        }
+        req.headers["Authorization"] = jwt.encode(
+            payload,
+            self.__key,
+            algorithm="ES256",
+            headers=headers,
+        )
+        return req