ingestr 0.12.5__py3-none-any.whl → 0.12.7__py3-none-any.whl

ingestr/main.py

@@ -444,7 +444,7 @@ def ingest(
 
         progressInstance: Collector = SpinnerCollector()
         if progress == Progress.log:
-            progressInstance = LogCollector(dump_system_stats=False)
+            progressInstance = LogCollector()
 
         is_pipelines_dir_temp = False
         if pipelines_dir is None:

ingestr/src/appstore/__init__.py

@@ -0,0 +1,137 @@
+import csv
+import gzip
+import os
+import tempfile
+from copy import deepcopy
+from datetime import datetime
+from typing import Iterable, List, Optional
+
+import dlt
+import requests
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from .client import AppStoreConnectClientInterface
+from .errors import (
+    NoOngoingReportRequestsFoundError,
+    NoReportsFoundError,
+    NoSuchReportError,
+)
+from .models import AnalyticsReportInstancesResponse
+from .resources import RESOURCES
+
+
+@dlt.source
+def app_store(
+    client: AppStoreConnectClientInterface,
+    app_ids: List[str],
+    start_date: Optional[datetime] = None,
+    end_date: Optional[datetime] = None,
+) -> Iterable[DltResource]:
+    for resource in RESOURCES:
+        yield dlt.resource(
+            get_analytics_reports,
+            name=resource.name,
+            primary_key=resource.primary_key,
+            columns=resource.columns,
+        )(client, app_ids, resource.report_name, start_date, end_date)
+
+
+def filter_instances_by_date(
+    instances: AnalyticsReportInstancesResponse,
+    start_date: Optional[datetime],
+    end_date: Optional[datetime],
+) -> AnalyticsReportInstancesResponse:
+    instances = deepcopy(instances)
+    if start_date is not None:
+        instances.data = list(
+            filter(
+                lambda x: datetime.fromisoformat(x.attributes.processingDate)
+                >= start_date,
+                instances.data,
+            )
+        )
+    if end_date is not None:
+        instances.data = list(
+            filter(
+                lambda x: datetime.fromisoformat(x.attributes.processingDate)
+                <= end_date,
+                instances.data,
+            )
+        )
+
+    return instances
+
+
+def get_analytics_reports(
+    client: AppStoreConnectClientInterface,
+    app_ids: List[str],
+    report_name: str,
+    start_date: Optional[datetime],
+    end_date: Optional[datetime],
+    last_processing_date=dlt.sources.incremental("processing_date"),
+) -> Iterable[TDataItem]:
+    if last_processing_date.last_value:
+        start_date = datetime.fromisoformat(last_processing_date.last_value)
+    for app_id in app_ids:
+        yield from get_report(client, app_id, report_name, start_date, end_date)
+
+
+def get_report(
+    client: AppStoreConnectClientInterface,
+    app_id: str,
+    report_name: str,
+    start_date: Optional[datetime],
+    end_date: Optional[datetime],
+) -> Iterable[TDataItem]:
+    report_requests = client.list_analytics_report_requests(app_id)
+    ongoing_requests = list(
+        filter(
+            lambda x: x.attributes.accessType == "ONGOING"
+            and not x.attributes.stoppedDueToInactivity,
+            report_requests.data,
+        )
+    )
+
+    if len(ongoing_requests) == 0:
+        raise NoOngoingReportRequestsFoundError()
+
+    reports = client.list_analytics_reports(ongoing_requests[0].id, report_name)
+    if len(reports.data) == 0:
+        raise NoSuchReportError(report_name)
+
+    for report in reports.data:
+        instances = client.list_report_instances(report.id)
+
+        instances = filter_instances_by_date(instances, start_date, end_date)
+
+        if len(instances.data) == 0:
+            raise NoReportsFoundError()
+
+        for instance in instances.data:
+            segments = client.list_report_segments(instance.id)
+            with tempfile.TemporaryDirectory() as temp_dir:
+                files = []
+                for segment in segments.data:
+                    payload = requests.get(segment.attributes.url, stream=True)
+                    payload.raise_for_status()
+
+                    csv_path = os.path.join(
+                        temp_dir, f"{segment.attributes.checksum}.csv"
+                    )
+                    with open(csv_path, "wb") as f:
+                        for chunk in payload.iter_content(chunk_size=8192):
+                            f.write(chunk)
+                    files.append(csv_path)
+                for file in files:
+                    with gzip.open(file, "rt") as f:
+                        # TODO: infer delimiter from the file itself
+                        delimiter = (
+                            "," if report_name == "App Crashes Expanded" else "\t"
+                        )
+                        reader = csv.DictReader(f, delimiter=delimiter)
+                        for row in reader:
+                            yield {
+                                "processing_date": instance.attributes.processingDate,
+                                **row,
+                            }

ingestr/src/appstore/client.py

@@ -0,0 +1,126 @@
+import abc
+import time
+from typing import Optional
+
+import jwt
+import requests
+from requests.models import PreparedRequest
+
+from .models import (
+    AnalyticsReportInstancesResponse,
+    AnalyticsReportRequestsResponse,
+    AnalyticsReportResponse,
+    AnalyticsReportSegmentsResponse,
+)
+
+
+class AppStoreConnectClientInterface(abc.ABC):
+    @abc.abstractmethod
+    def list_analytics_report_requests(self, app_id) -> AnalyticsReportRequestsResponse:
+        pass
+
+    @abc.abstractmethod
+    def list_analytics_reports(
+        self, req_id: str, report_name: str
+    ) -> AnalyticsReportResponse:
+        pass
+
+    @abc.abstractmethod
+    def list_report_instances(
+        self,
+        report_id: str,
+        granularity: str = "DAILY",
+    ) -> AnalyticsReportInstancesResponse:
+        pass
+
+    @abc.abstractmethod
+    def list_report_segments(self, instance_id: str) -> AnalyticsReportSegmentsResponse:
+        pass
+
+
+class AppStoreConnectClient(AppStoreConnectClientInterface):
+    def __init__(self, key: bytes, key_id: str, issuer_id: str):
+        self.__key = key
+        self.__key_id = key_id
+        self.__issuer_id = issuer_id
+
+    def list_analytics_report_requests(self, app_id) -> AnalyticsReportRequestsResponse:
+        res = requests.get(
+            f"https://api.appstoreconnect.apple.com/v1/apps/{app_id}/analyticsReportRequests",
+            auth=self.auth,
+        )
+        res.raise_for_status()
+
+        return AnalyticsReportRequestsResponse.from_json(res.text)  # type: ignore
+
+    def list_analytics_reports(
+        self, req_id: str, report_name: str
+    ) -> AnalyticsReportResponse:
+        params = {"filter[name]": report_name}
+        res = requests.get(
+            f"https://api.appstoreconnect.apple.com/v1/analyticsReportRequests/{req_id}/reports",
+            auth=self.auth,
+            params=params,
+        )
+        res.raise_for_status()
+        return AnalyticsReportResponse.from_json(res.text)  # type: ignore
+
+    def list_report_instances(
+        self,
+        report_id: str,
+        granularity: str = "DAILY",
+    ) -> AnalyticsReportInstancesResponse:
+        data = []
+        url = f"https://api.appstoreconnect.apple.com/v1/analyticsReports/{report_id}/instances"
+        params: Optional[dict] = {"filter[granularity]": granularity}
+
+        while url:
+            res = requests.get(url, auth=self.auth, params=params)
+            res.raise_for_status()
+
+            response_data = AnalyticsReportInstancesResponse.from_json(res.text)  # type: ignore
+            data.extend(response_data.data)
+
+            url = response_data.links.next
+            params = None  # Clear params for subsequent requests
+
+        return AnalyticsReportInstancesResponse(
+            data=data,
+            links=response_data.links,
+            meta=response_data.meta,
+        )
+
+    def list_report_segments(self, instance_id: str) -> AnalyticsReportSegmentsResponse:
+        segments = []
+        url = f"https://api.appstoreconnect.apple.com/v1/analyticsReportInstances/{instance_id}/segments"
+
+        while url:
+            res = requests.get(url, auth=self.auth)
+            res.raise_for_status()
+
+            response_data = AnalyticsReportSegmentsResponse.from_json(res.text)  # type: ignore
+            segments.extend(response_data.data)
+
+            url = response_data.links.next
+
+        return AnalyticsReportSegmentsResponse(
+            data=segments, links=response_data.links, meta=response_data.meta
+        )
+
+    def auth(self, req: PreparedRequest) -> PreparedRequest:
+        headers = {
+            "alg": "ES256",
+            "kid": self.__key_id,
+        }
+        payload = {
+            "iss": self.__issuer_id,
+            "exp": int(time.time()) + 600,
+            "aud": "appstoreconnect-v1",
+        }
+        req.headers["Authorization"] = jwt.encode(
+            payload,
+            self.__key,
+            algorithm="ES256",
+            headers=headers,
+        )
+        return req

ingestr/src/appstore/errors.py

@@ -0,0 +1,15 @@
+class NoReportsFoundError(Exception):
+    def __init__(self):
+        super().__init__("No Report instances found for the given date range")
+
+
+class NoOngoingReportRequestsFoundError(Exception):
+    def __init__(self):
+        super().__init__(
+            "No ONGOING report requests found (or they're stopped due to inactivity)"
+        )
+
+
+class NoSuchReportError(Exception):
+    def __init__(self, report_name):
+        super().__init__(f"No such report found: {report_name}")

ingestr/src/appstore/models.py

@@ -0,0 +1,117 @@
+from dataclasses import dataclass
+from typing import List, Optional
+
+from dataclasses_json import dataclass_json
+
+
+@dataclass_json
+@dataclass
+class Links:
+    self: str
+    next: Optional[str] = None
+
+
+@dataclass_json
+@dataclass
+class ReportRequestAttributes:
+    accessType: str
+    stoppedDueToInactivity: bool
+
+
+@dataclass_json
+@dataclass
+class ReportAttributes:
+    name: str
+    category: str
+
+
+@dataclass_json
+@dataclass
+class ReportInstanceAttributes:
+    granularity: str
+    processingDate: str
+
+
+@dataclass_json
+@dataclass
+class ReportSegmentAttributes:
+    checksum: str
+    url: str
+    sizeInBytes: int
+
+
+@dataclass_json
+@dataclass
+class ReportRequest:
+    type: str
+    id: str
+    attributes: ReportRequestAttributes
+
+
+@dataclass_json
+@dataclass
+class Report:
+    type: str
+    id: str
+    attributes: ReportAttributes
+
+
+@dataclass_json
+@dataclass
+class ReportInstance:
+    type: str
+    id: str
+    attributes: ReportInstanceAttributes
+
+
+@dataclass_json
+@dataclass
+class ReportSegment:
+    type: str
+    id: str
+    attributes: ReportSegmentAttributes
+
+
+@dataclass_json
+@dataclass
+class PagingMeta:
+    total: int
+    limit: int
+
+
+@dataclass_json
+@dataclass
+class Meta:
+    paging: PagingMeta
+
+
+@dataclass_json
+@dataclass
+class AnalyticsReportRequestsResponse:
+    data: List[ReportRequest]
+    meta: Meta
+    links: Links
+
+
+@dataclass_json
+@dataclass
+class AnalyticsReportResponse:
+    data: List[Report]
+    meta: Meta
+    links: Links
+
+
+@dataclass_json
+@dataclass
+class AnalyticsReportInstancesResponse:
+    data: List[ReportInstance]
+    meta: Meta
+    links: Links
+
+
+@dataclass_json
+@dataclass
+class AnalyticsReportSegmentsResponse:
+    data: List[ReportSegment]
+    meta: Meta
+    links: Links

ingestr/src/appstore/resources.py

@@ -0,0 +1,179 @@
+from dataclasses import dataclass
+from typing import List
+
+
+@dataclass
+class ResourceConfig:
+    name: str
+    primary_key: List[str]
+    columns: dict
+    report_name: str
+
+
+RESOURCES: List[ResourceConfig] = [
+    ResourceConfig(
+        name="app-downloads-detailed",
+        primary_key=[
+            "App Apple Identifier",
+            "App Name",
+            "App Version",
+            "Campaign",
+            "Date",
+            "Device",
+            "Download Type",
+            "Page Title",
+            "Page Type",
+            "Platform Version",
+            "Pre-Order",
+            "Source Info",
+            "Source Type",
+            "Territory",
+        ],
+        columns={
+            "Date": {"data_type": "date"},
+            "App Apple Identifier": {"data_type": "bigint"},
+            "Counts": {"data_type": "bigint"},
+            "processing_date": {"data_type": "date"},
+        },
+        report_name="App Downloads Detailed",
+    ),
+    ResourceConfig(
+        name="app-store-discovery-and-engagement-detailed",
+        primary_key=[
+            "App Apple Identifier",
+            "App Name",
+            "Campaign",
+            "Date",
+            "Device",
+            "Engagement Type",
+            "Event",
+            "Page Title",
+            "Page Type",
+            "Platform Version",
+            "Source Info",
+            "Source Type",
+            "Territory",
+        ],
+        columns={
+            "Date": {"data_type": "date"},
+            "App Apple Identifier": {"data_type": "bigint"},
+            "Counts": {"data_type": "bigint"},
+            "Unique Counts": {"data_type": "bigint"},
+            "processing_date": {"data_type": "date"},
+        },
+        report_name="App Store Discovery and Engagement Detailed",
+    ),
+    ResourceConfig(
+        name="app-sessions-detailed",
+        primary_key=[
+            "Date",
+            "App Name",
+            "App Apple Identifier",
+            "App Version",
+            "Device",
+            "Platform Version",
+            "Source Type",
+            "Source Info",
+            "Campaign",
+            "Page Type",
+            "Page Title",
+            "App Download Date",
+            "Territory",
+        ],
+        columns={
+            "Date": {"data_type": "date"},
+            "App Apple Identifier": {"data_type": "bigint"},
+            "Sessions": {"data_type": "bigint"},
+            "Total Session Duration": {"data_type": "bigint"},
+            "Unique Devices": {"data_type": "bigint"},
+            "processing_date": {"data_type": "date"},
+        },
+        report_name="App Sessions Detailed",
+    ),
+    ResourceConfig(
+        name="app-store-installation-and-deletion-detailed",
+        primary_key=[
+            "App Apple Identifier",
+            "App Download Date",
+            "App Name",
+            "App Version",
+            "Campaign",
+            "Counts",
+            "Date",
+            "Device",
+            "Download Type",
+            "Event",
+            "Page Title",
+            "Page Type",
+            "Platform Version",
+            "Source Info",
+            "Source Type",
+            "Territory",
+            "Unique Devices",
+        ],
+        columns={
+            "Date": {"data_type": "date"},
+            "App Apple Identifier": {"data_type": "bigint"},
+            "Counts": {"data_type": "bigint"},
+            "Unique Devices": {"data_type": "bigint"},
+            "App Download Date": {"data_type": "date"},
+            "processing_date": {"data_type": "date"},
+        },
+        report_name="App Store Installation and Deletion Detailed",
+    ),
+    ResourceConfig(
+        name="app-store-purchases-detailed",
+        primary_key=[
+            "App Apple Identifier",
+            "App Download Date",
+            "App Name",
+            "Campaign",
+            "Content Apple Identifier",
+            "Content Name",
+            "Date",
+            "Device",
+            "Page Title",
+            "Page Type",
+            "Payment Method",
+            "Platform Version",
+            "Pre-Order",
+            "Purchase Type",
+            "Source Info",
+            "Source Type",
+            "Territory",
+        ],
+        columns={
+            "Date": {"data_type": "date"},
+            "App Apple Identifier": {"data_type": "bigint"},
+            "App Download Date": {"data_type": "date"},
+            "Content Apple Identifier": {"data_type": "bigint"},
+            "Purchases": {"data_type": "bigint"},
+            "Proceeds In USD": {"data_type": "double"},
+            "Sales In USD": {"data_type": "double"},
+            "Paying Users": {"data_type": "bigint"},
+            "processing_date": {"data_type": "date"},
+        },
+        report_name="App Store Purchases Detailed",
+    ),
+    ResourceConfig(
+        name="app-crashes-expanded",
+        primary_key=[
+            "App Name",
+            "App Version",
+            "Build",
+            "Date",
+            "Device",
+            "Platform",
+            "Release Type",
+            "Territory",
+        ],
+        columns={
+            "Date": {"data_type": "date"},
+            "processing_date": {"data_type": "date"},
+            "App Apple Identifier": {"data_type": "bigint"},
+            "Count": {"data_type": "bigint"},
+            "Unique Devices": {"data_type": "bigint"},
+        },
+        report_name="App Crashes Expanded",
+    ),
+]

ingestr/src/asana_source/__init__.py

@@ -150,7 +150,10 @@ def tasks(
     project_array: t.List[TDataItem],
     access_token: str = dlt.secrets.value,
     modified_at: dlt.sources.incremental[str] = dlt.sources.incremental(
-        "modified_at", initial_value=DEFAULT_START_DATE
+        "modified_at",
+        initial_value=DEFAULT_START_DATE,
+        range_end="closed",
+        range_start="closed",
     ),
     fields: Iterable[str] = TASK_FIELDS,
 ) -> Iterable[TDataItem]:

ingestr/src/errors.py

@@ -0,0 +1,10 @@
+class MissingValueError(Exception):
+    def __init__(self, value, source):
+        super().__init__(f"{value} is required to connect to {source}")
+
+
+class UnsupportedResourceError(Exception):
+    def __init__(self, resource, source):
+        super().__init__(
+            f"Resource '{resource}' is not supported for {source} source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+        )

ingestr/src/facebook_ads/__init__.py

@@ -159,7 +159,10 @@ def facebook_insights_source(
     )
     def facebook_insights(
         date_start: dlt.sources.incremental[str] = dlt.sources.incremental(
-            "date_start", initial_value=initial_load_start_date_str
+            "date_start",
+            initial_value=initial_load_start_date_str,
+            range_end="closed",
+            range_start="closed",
         ),
     ) -> Iterator[TDataItems]:
         start_date = get_start_date(date_start, attribution_window_days_lag)

ingestr/src/factory.py

@@ -18,6 +18,7 @@ from ingestr.src.destinations import (
 from ingestr.src.sources import (
     AdjustSource,
     AirtableSource,
+    AppleAppStoreSource,
     AppsflyerSource,
     ArrowMemoryMappedSource,
     AsanaSource,
@@ -122,6 +123,7 @@ class SourceDestinationFactory:
         "asana": AsanaSource,
         "tiktok": TikTokSource,
         "googleanalytics": GoogleAnalyticsSource,
+        "appstore": AppleAppStoreSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr/src/filesystem/__init__.py

@@ -38,7 +38,9 @@ def readers(
     """
     filesystem_resource = filesystem(bucket_url, credentials, file_glob=file_glob)
     filesystem_resource.apply_hints(
-        incremental=dlt.sources.incremental("modification_date")
+        incremental=dlt.sources.incremental("modification_date"),
+        range_end="closed",
+        range_start="closed",
     )
     return (
         filesystem_resource | dlt.transformer(name="read_csv")(_read_csv),

ingestr/src/github/__init__.py

@@ -14,7 +14,7 @@ from .helpers import get_reactions_data, get_rest_pages, get_stargazers
 def github_reactions(
     owner: str,
     name: str,
-    access_token: str = dlt.secrets.value,
+    access_token: str,
     items_per_page: int = 100,
     max_items: Optional[int] = None,
 ) -> Sequence[DltResource]:
@@ -89,7 +89,11 @@ def github_repo_events(
     @dlt.resource(primary_key="id", table_name=lambda i: i["type"])
     def repo_events(
         last_created_at: dlt.sources.incremental[str] = dlt.sources.incremental(
-            "created_at", initial_value="1970-01-01T00:00:00Z", last_value_func=max
+            "created_at",
+            initial_value="1970-01-01T00:00:00Z",
+            last_value_func=max,
+            range_end="closed",
+            range_start="closed",
         ),
     ) -> Iterator[TDataItems]:
         repos_path = (
@@ -114,7 +118,7 @@ def github_repo_events(
 def github_stargazers(
     owner: str,
     name: str,
-    access_token: str = dlt.secrets.value,
+    access_token: str,
     items_per_page: int = 100,
     max_items: Optional[int] = None,
 ) -> Sequence[DltResource]: