ingestr 0.12.6__tar.gz → 0.12.8__tar.gz

{ingestr-0.12.6 → ingestr-0.12.8}/.githooks/pre-commit-hook.sh

@@ -13,11 +13,11 @@ secret_detected() {
 
 # use gitleaks binary if available
 # else fallback to using docker for running gitleaks
-CMD="gitleaks dir -v"
+CMD="gitleaks protect --staged -v"
 
 if [[ ! `which gitleaks`  ]]; then
     which docker > /dev/null || (echo "gitleaks or docker is required for running secrets scan." && exit 1)
-    CMD="docker run -v $PWD:$WORK_DIR -w $WORK_DIR --rm ghcr.io/gitleaks/gitleaks:latest dir -v"
+    CMD="docker run -v $PWD:$WORK_DIR -w $WORK_DIR --rm ghcr.io/gitleaks/gitleaks:latest protect --staged -v"
 fi
 
 $CMD || secret_detected

{ingestr-0.12.6 → ingestr-0.12.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.12.6
+Version: 0.12.8
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -17,6 +17,7 @@ Requires-Python: >=3.9
 Requires-Dist: asana==3.2.3
 Requires-Dist: confluent-kafka>=2.6.1
 Requires-Dist: databricks-sql-connector==2.9.3
+Requires-Dist: dataclasses-json==0.6.7
 Requires-Dist: dlt==1.5.0
 Requires-Dist: duckdb-engine==0.13.5
 Requires-Dist: duckdb==1.1.3

{ingestr-0.12.6 → ingestr-0.12.8}/docs/.vitepress/config.mjs

@@ -94,6 +94,7 @@ export default defineConfig({
               { text: "Adjust", link: "/supported-sources/adjust.md" },
               { text: "Airtable", link: "/supported-sources/airtable.md" },
               { text: "AppsFlyer", link: "/supported-sources/appsflyer.md" },
+              { text: "Apple App Store", link: "/supported-sources/appstore.md"},
               { text: "Asana", link: "/supported-sources/asana.md" },
               { text: "Chess.com", link: "/supported-sources/chess.md" },
               { text: "DynamoDB", link: "/supported-sources/dynamodb.md" },

{ingestr-0.12.6 → ingestr-0.12.8}/docs/supported-sources/adjust.md

@@ -34,6 +34,7 @@ Adjust source allows ingesting data from various sources:
 
 - `campaigns`: Retrieves data for a campaign, showing the app's revenue and network costs over multiple days.
 - `creatives`: Retrieves data for a creative assets, detailing the app's revenue and network costs across multiple days.
+- `events`: Retrieves data for [events](https://dev.adjust.com/en/api/rs-api/events/) and event slugs.
 - `custom`: Retrieves custom data based on the dimensions and metrics specified.
 
 ### Custom reports: `custom:<dimensions>:<metrics>[:<filters>]`

ingestr-0.12.8/docs/supported-sources/appstore.md

@@ -0,0 +1,254 @@
+# Apple App Store 
+The [App Store](https://appstore.com/) is an app marketplace developed and maintained by Apple, for mobile apps on its iOS and iPadOS operating systems. The store allows users to browse and download approved apps developed within Apple's iOS SDK. Apps can be downloaded on the iPhone, iPod Touch, or iPad, and some can be transferred to the Apple Watch smartwatch or 4th-generation or newer Apple TVs as extensions of iPhone apps.
+
+`ingestr` allows you to ingest analytics, sales and performance data using the [Apple App Store Connect API](https://developer.apple.com/documentation/appstoreconnectapi)
+
+> [!NOTE]
+> Sometimes, the data in App Store Analytics reports isn’t fully complete when first provided. This happens because some information takes longer to process and appears in the reports later. For example, certain usage or sales details might be updated after the initial report is generated to correct errors or include missing data. This means that the report for a certain date may include data points from older dates. `ingestr` takes care of updating these rows to show the updated values. However, caution should be exercised when analysing current date's data, as it maybe subject to change in the future. 
+> see [Data Completeness and Corrections](https://developer.apple.com/documentation/analytics-reports/data-completeness-corrections) for more information.
+## URI Format
+
+The URI format for App Store is as follows:
+```
+appstore://?key_path=</path/to/key>&key_id=<key_id>&issuer_id=<issuer_id>&app_id=<app_id>
+```
+
+URI Parameters:
+* `key_path`: path to API private key
+* `key_id`: ID of the generated key
+* `issuer_id`: Issuer ID of the generated key
+* `app_id`: optional, application ID of your app. You can specify `app_id` multiple times with different ids to ingest data for multiple apps.
+  * You can also define the app_id in the table name. For example, `app-downloads-detailed:12345,67890` will ingest data for app with id `12345` and `67890`.
+
+## Setting up Appstore Integration
+
+
+### Prerequisites
+To generate an API key, you must have an Admin account in App Store Connect. 
+
+### Generate an API Key
+
+To generate a new API key to use with `ingestr`, log in to [App Store Connect](https://appstoreconnect.apple.com/) and:
+
+1. Select Users and Access, and then select the API Keys tab.
+2. Make sure the Team Keys tab is selected.
+3. Click Generate API Key or the Add (+) button.
+4. Enter a name for the key. The name is for your reference only and isn’t part of the key itself.
+5. Under Access, select the role as `FINANCE`.
+6. Click Generate.
+
+The new key’s name, key ID, a download link, and other information appears on the page.
+
+For more information, see [App Store Connect docs](https://developer.apple.com/documentation/appstoreconnectapi/creating-api-keys-for-app-store-connect-api)
+
+### Find your Apps ID
+
+You can find the App ID of your app by:
+1. Opening the app entry in App Store Connect
+2. Looking for "General Information" in the App information tab
+3. Finding your App ID under the "Apple ID" entry
+
+With this, you are ready to ingest data from App Store.
+
+### Request a Report.
+Before you can ingest analytics data from App Store, you need to submit a [Report Request](https://developer.apple.com/documentation/appstoreconnectapi/post-v1-analyticsreportrequests). See [App Store Connect docs](https://developer.apple.com/documentation/appstoreconnectapi/downloading-analytics-reports) for more information on how to Request a Report.
+
+We recommend using `ONGOING` access-type for reports. Please note that it may take upto 48 hours after submitting a Report Request for the data to become available. For more information, see [Request analytics report](https://developer.apple.com/documentation/appstoreconnectapi/downloading-analytics-reports#Request-analytics-reports).
+
+> [!NOTE]
+> you have to create a Report Request for each individual App that you want to ingest data for. You can use [list apps](https://developer.apple.com/documentation/appstoreconnectapi/get-v1-apps) API to get the list of all apps in your Apple Account.
+### Example: Loading App Downloads Analytics
+
+For this example, we'll assume that:
+* `key_id` is `key_0`
+* `issuer_id` is `issue_0`
+* `key` is stored in the current directory and is named `api.key`
+* `app_id` is `12345`
+
+We will run `ingestr` to save this data to a [duckdb](https://duckdb.org/) database called `analytics.db` under the name `public.app_downloads`.
+
+```sh
+ingestr ingest \
+    --source-uri "appstore://app_id=12345&key_path=api.key&key_id=key_0&issuer_id=issue_0 \
+    --source-table "app-downloads-detailed" \
+    --dest-uri "duckdb:///analytics.db"  \
+    --dest-table "public.app_downloads" \
+```
+
+### Example: Loading Data for multiple Apps
+
+We will extend the prior example with another app with ID `67890`. To achieve this, simply add another `app_id` query parameter to the URI.
+```sh
+ingestr ingest \
+    --source-uri "appstore://app_id=12345&app_id=67890&key_path=api.key&key_id=key_0&issuer_id=issue_0 \
+    --source-table "app-downloads-detailed" \
+    --dest-uri "duckdb:///analytics.db"  \
+    --dest-table "public.app_downloads" \
+```
+
+
+### Example: Incremental Loading
+
+`ingestr` supports incremental loading for all App Store tables.
+
+To begin, we will first load all data till `2025-01-01` by specifying the `--interval-end` flag. We'll assume the same credentials from our [first example](#example-loading-app-downloads-analytics)
+```sh
+ingestr ingest \
+    --source-uri "appstore://app_id=12345&key_path=api.key&key_id=key_0&issuer_id=issue_0 \
+    --source-table "app-downloads-detailed" \
+    --dest-uri "duckdb:///analytics.db"  \
+    --dest-table "public.app_downloads" \
+    --interval-end "2025-01-01"
+```
+
+`ingestr` will load all data available till `2025-01-01`. Now we will run `ingestr` again, but this time, we'll let `ingestr` pickup from where it left off by specifying the `--incremental-strategy` flag.
+
+```sh
+ingestr ingest \
+    --source-uri "appstore://app_id=12345&key_path=api.key&key_id=key_0&issuer_id=issue_0 \
+    --source-table "app-downloads-detailed" \
+    --dest-uri "duckdb:///analytics.db"  \
+    --dest-table "public.app_downloads" \
+    --incremental-strategy "merge"
+```
+
+Notice how we didn't specify a date parameter? `ingestr` will automatically use the metadata from last load and continue loading data from that point on.
+
+## Tables
+
+### `app-downloads-detailed`
+The App Downloads Report includes download data generated on the App Store. You can use this report to understand your total number of downloads, including first-time downloads, redownloads, updates, and more.
+
+| **Column**   | **Description** |
+|--------------|-----------------|
+| `date`  | Date on which the event occurred. |
+| `app_name`    | The name of the app provided by you during app setup in App Store Connect.|
+| `app_apple_identifier`    |  Your app’s Apple ID.  |
+| `download_type`    | The type of download event that occured. |
+| `app_version` | The app version being downloaded. | 
+| `device`     | The device on which the app was downloaded.|
+| `platform_version` | The OS version of the device on which the download occured.|
+| `source_type` | The source from where the user discovered the app.|
+| `source_info` | The app referrer or web referrer that led the user to discover the app.|
+| `campaign` | The Campaign Token of the campaign created in App Analytics. Column available starting November 19, 2024.|
+| `page_type` | The page type from where the app was downloaded. |
+| `page_title` | The name of the product page or in-app event page that led the user to download the app.|
+| `pre-order` | A flag indicating whether the download came from a pre-order.|
+| `territory` | The App Store country or region where the download occured.|
+| `counts` | The total number of downloads.|
+
+
+### `app-store-discovery-and-engagement-detailed`
+The App Store Discovery and Engagement report provides details about how users engage with your apps on the App Store itself. This includes data about user engagement with your app’s icons, product pages, in-app event pages, and other install sheets.
+
+
+| **Column**   | **Description** |
+|--------------|-----------------|
+| `date`  | Date on which the event occurred. |
+| `app_name`    | The name of the app provided by you during app setup in App Store Connect.|
+| `app_apple_identifier`    |  Your app’s Apple ID.  |
+| `event`    | The type of event that occurred.|
+| `source_type` | The source from where the user discovered the app. |
+| `source_info` | The app referrer or web referrer that led the user to discover the app.|
+| `campaign` | The Campaign Token of the campaign created in App Analytics. Column available starting November 19, 2024.|
+| `page_type` | The page type from where the app was downloaded. |
+| `page_title` | The name of the product page or in-app event page that led the user to download the app.|
+| `device` | The device on which the event occurred. |
+| `engagement_type` | User action, if any, on the impression or page.|
+| `platform_version` | The OS version of the device on which the event occurred.|
+| `territory` | The App Store country or region where the download occured.|
+| `counts` | The total number of events that occurred. |
+| `unique_counts` | The total number of unique users that performed the event.|
+
+### `app-sessions-detailed`
+App Session provides insights on how often people open your app, and how long they spend in your app.
+
+| **Column**   | **Description** |
+|--------------|-----------------|
+| `date`  | Date on which the event occurred. |
+| `app_name`    | The name of the app provided by you during app setup in App Store Connect.|
+| `app_apple_identifier`    |  Your app’s Apple ID.  |
+| `download_type`    | The type of download event that occured. |
+| `app_version` | The app version being downloaded. | 
+| `device`     | The device on which the app was downloaded.|
+| `platform_version` | The OS version of the device on which the download occured.|
+| `source_type` | The source from where the user discovered the app.|
+| `source_info` | The app referrer or web referrer that led the user to discover the app.|
+| `campaign` | The Campaign Token of the campaign created in App Analytics. Column available starting November 19, 2024.|
+| `page_type` | The page type from where the app was downloaded. |
+| `page_title` | The name of the product page or in-app event page that led the user to download the app.|
+| `app_download_date` | The date on which the app was downloaded onto the device. This field is only populated if the download occurred in the previous 30 days, otherwise it is null.|
+| `territory` | The App Store country or region where the download occured.|
+| `sessions` | The number of sessions. Based on users who have agreed to share their data with Apple and developers.|
+| `total_session_duration` | The total duration, in seconds, of all sessions being reported.|
+| `unique_devices` | The number of unique devices contributing to the total number of sessions being reported.|
+
+### `app-store-installation-and-deletion-detailed`
+Use the data in App Store Installation and Deletion report to estimate the number of times people install and delete your App Store apps.
+
+
+| **Column**   | **Description** |
+|--------------|-----------------|
+| `date`| Date on which the event occurred.|
+| `app_name`| The name of the app provided by you during app setup in App Store Connect.|
+| `app_apple_identifier`| Date on which the event occurred.|
+| `event`| The type of usage event that occurred. |
+| `download_type`| The type of download event that occurred.|
+| `app_version`| The version of the app being associated with the instalation or deletion.|
+| `device`| The device on which the app was installed or deleted.|
+| `platform_version`| The OS version of the device on which the app was installed or deleted.|
+| `source_type`| Where the user discovered your app.|
+| `source_info`| The app referrer or web referrer that led the user to discover your app.|
+| `campaign`| The Campaign Token of the campaign created in App Analytics. Column available starting November 19, 2024.|
+| `page_type`| The page type which led the user to discover your app.|
+| `page_title`| The name of the product page or in-app event page that led the user to discover your app.|
+| `app_download_date`| The date on which the app was downloaded onto the device. This field is only populated if the download occurred in the previous 30 days, otherwise it is null.|
+| `territory`| The App Store country or region where the installation or deletion occurred.|
+| `counts`| The total count of events, based on users who have agreed to share their data with Apple and developers.|
+| `unique_devices`| The number of unique devices on which events were generated, based on users who have agreed to share their data with Apple and developers.|
+
+### `app-store-purchases-detailed`
+The App Store Purchases Report includes App Store paid app and in-app purchase data. Using the data in this report, you can measure your total revenue generated on the App Store, attribute sales to download sources and page types, and measure how many paying users you have for each individual row. Paying user counts are not summable across rows, because the same user can exist in multiple rows.
+
+| **Column**   | **Description** |
+|--------------|-----------------|
+| `date`| Date on which the event occurred.|
+| `app_name`| The name of the app provided by you during app setup in App Store Connect.|
+| `app_apple_identifier`| Your app’s Apple ID.|
+| `purchase_type`| The type of purchase made by the user on the App Store.|
+| `content_name`| The name of the content being purchased. For paid apps, the field will populate the name of app as set in App Store. For in-app purchases, the field will populate the name of the SKU as set in App Store Connect.|
+| `content_apple_identifier`| Your content’s Apple ID |
+| `payment_method`| The payment type used to charge the customer.|
+| `device`| The device on which the purchase occurred.|
+| `platform_version`| The OS version of the device on which the app was installed or deleted.|
+| `source_type`| Where the user discovered your app.|
+| `source_info`| The app referrer or web referrer that led the user to discover your app.|
+| `campaign`| The Campaign Token of the campaign created in App Analytics. Column available starting November 19, 2024.|
+| `page_type`| The page type which led the user to discover your app.|
+| `page_title`| The name of the product page or in-app event page that led the user to discover your app.|
+| `app_download_date`| The date on which the app was downloaded onto the device. This field is only populated if the download occurred in the previous 30 days, otherwise it is null.|
+| `pre-order`| Indicates whether the purchase originated from someone who pre-ordered the app.|
+| `territory`| The App Store country or region in which the purchase occurred.|
+| `purchases`| Aggregated count of purchases made. Negative value indicates refunds. If purchases count is 0 and proceeds, and sales are negative, it indicates partial refunds.|
+| `proceeds_in_usd`| The estimated proceeds in USD from purchases of your app and in-app purchases. This is the Customer Price minus applicable taxes and Apple’s commission, per Schedule 2 of the Paid Apps Agreement.|
+| `sales_in_usd`| The estimated sales in USD from purchases of your app and in-app purchases.|
+| `paying_users`| The number of unique users who paid for your app or in-app purchases. This metric is not summable across rows.|
+
+### `app-crashes-expanded`
+Use this report to understand crashes for your App Store apps by app version and device type.
+
+| **Column**   | **Description** |
+|--------------|-----------------|
+| `date`| Date on which the event occurred.|
+| `app_name`| The name of the app provided by you during app setup in App Store Connect.|
+| `app_apple_identifier`| Your app’s Apple ID.|
+| `app_version` | The app version being downloaded. | 
+| `device`     | The device on which the app was downloaded.|
+| `platform_version` | The OS version of the device on which the download occured.|
+| `crashes` | The total number of crashes.|
+| `unique_devices` | Number of unique devices where app crashed. |
+
+
+Use these as `--source-table` parameter in the `ingestr ingest` command.
+
+To know more about these reports and their dimensions, see [App Store Analytics docs](https://developer.apple.com/documentation/analytics-reports).

{ingestr-0.12.6 → ingestr-0.12.8}/ingestr/src/adjust/__init__.py

@@ -56,6 +56,11 @@ def adjust_source(
             filters=filters,
         )
 
+    @dlt.resource(write_disposition="replace", primary_key="id")
+    def events():
+        adjust_api = AdjustAPI(api_key=api_key)
+        yield adjust_api.fetch_events()
+
     @dlt.resource(write_disposition="merge", merge_key="day")
     def creatives():
         adjust_api = AdjustAPI(api_key=api_key)
@@ -68,7 +73,7 @@ def adjust_source(
         )
 
     if not dimensions:
-        return campaigns, creatives
+        return campaigns, creatives, events
 
     merge_key = merge_key
     type_hints = {}
@@ -100,4 +105,4 @@ def adjust_source(
             filters=filters,
         )
 
-    return campaigns, creatives, custom
+    return campaigns, creatives, custom, events

{ingestr-0.12.6 → ingestr-0.12.8}/ingestr/src/adjust/adjust_helpers.py

@@ -28,10 +28,20 @@ DEFAULT_METRICS = [
 ]
 
 
+def retry_on_limit(response: requests.Response, exception: BaseException) -> bool:
+    return response.status_code == 429
+
+
 class AdjustAPI:
     def __init__(self, api_key):
         self.api_key = api_key
-        self.uri = "https://automate.adjust.com/reports-service/report"
+        self.request_client = Client(
+            request_timeout=8.0,
+            raise_for_status=False,
+            retry_condition=retry_on_limit,
+            request_max_attempts=12,
+            request_backoff_factor=2,
+        ).session
 
     def fetch_report_data(
         self,
@@ -62,20 +72,11 @@ class AdjustAPI:
                 f"Invalid date range: Start date ({start_date}) must be earlier than end date ({end_date})."
             )
 
-        def retry_on_limit(
-            response: requests.Response, exception: BaseException
-        ) -> bool:
-            return response.status_code == 429
-
-        request_client = Client(
-            request_timeout=8.0,
-            raise_for_status=False,
-            retry_condition=retry_on_limit,
-            request_max_attempts=12,
-            request_backoff_factor=2,
-        ).session
-
-        response = request_client.get(self.uri, headers=headers, params=params)
+        response = self.request_client.get(
+            "https://automate.adjust.com/reports-service/report",
+            headers=headers,
+            params=params,
+        )
         if response.status_code == 200:
             result = response.json()
             items = result.get("rows", [])
@@ -83,6 +84,17 @@ class AdjustAPI:
         else:
             raise HTTPError(f"Request failed with status code: {response.status_code}")
 
+    def fetch_events(self):
+        headers = {"Authorization": f"Bearer {self.api_key}"}
+        response = self.request_client.get(
+            "https://automate.adjust.com/reports-service/events", headers=headers
+        )
+        if response.status_code == 200:
+            result = response.json()
+            yield result
+        else:
+            raise HTTPError(f"Request failed with status code: {response.status_code}")
+
 
 def parse_filters(filters_raw: str) -> dict:
     # Parse filter string like "key1=value1,key2=value2,value3,value4"

ingestr-0.12.8/ingestr/src/appstore/__init__.py

@@ -0,0 +1,137 @@
+import csv
+import gzip
+import os
+import tempfile
+from copy import deepcopy
+from datetime import datetime
+from typing import Iterable, List, Optional
+
+import dlt
+import requests
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from .client import AppStoreConnectClientInterface
+from .errors import (
+    NoOngoingReportRequestsFoundError,
+    NoReportsFoundError,
+    NoSuchReportError,
+)
+from .models import AnalyticsReportInstancesResponse
+from .resources import RESOURCES
+
+
+@dlt.source
+def app_store(
+    client: AppStoreConnectClientInterface,
+    app_ids: List[str],
+    start_date: Optional[datetime] = None,
+    end_date: Optional[datetime] = None,
+) -> Iterable[DltResource]:
+    for resource in RESOURCES:
+        yield dlt.resource(
+            get_analytics_reports,
+            name=resource.name,
+            primary_key=resource.primary_key,
+            columns=resource.columns,
+        )(client, app_ids, resource.report_name, start_date, end_date)
+
+
+def filter_instances_by_date(
+    instances: AnalyticsReportInstancesResponse,
+    start_date: Optional[datetime],
+    end_date: Optional[datetime],
+) -> AnalyticsReportInstancesResponse:
+    instances = deepcopy(instances)
+    if start_date is not None:
+        instances.data = list(
+            filter(
+                lambda x: datetime.fromisoformat(x.attributes.processingDate)
+                >= start_date,
+                instances.data,
+            )
+        )
+    if end_date is not None:
+        instances.data = list(
+            filter(
+                lambda x: datetime.fromisoformat(x.attributes.processingDate)
+                <= end_date,
+                instances.data,
+            )
+        )
+
+    return instances
+
+
+def get_analytics_reports(
+    client: AppStoreConnectClientInterface,
+    app_ids: List[str],
+    report_name: str,
+    start_date: Optional[datetime],
+    end_date: Optional[datetime],
+    last_processing_date=dlt.sources.incremental("processing_date"),
+) -> Iterable[TDataItem]:
+    if last_processing_date.last_value:
+        start_date = datetime.fromisoformat(last_processing_date.last_value)
+    for app_id in app_ids:
+        yield from get_report(client, app_id, report_name, start_date, end_date)
+
+
+def get_report(
+    client: AppStoreConnectClientInterface,
+    app_id: str,
+    report_name: str,
+    start_date: Optional[datetime],
+    end_date: Optional[datetime],
+) -> Iterable[TDataItem]:
+    report_requests = client.list_analytics_report_requests(app_id)
+    ongoing_requests = list(
+        filter(
+            lambda x: x.attributes.accessType == "ONGOING"
+            and not x.attributes.stoppedDueToInactivity,
+            report_requests.data,
+        )
+    )
+
+    if len(ongoing_requests) == 0:
+        raise NoOngoingReportRequestsFoundError()
+
+    reports = client.list_analytics_reports(ongoing_requests[0].id, report_name)
+    if len(reports.data) == 0:
+        raise NoSuchReportError(report_name)
+
+    for report in reports.data:
+        instances = client.list_report_instances(report.id)
+
+        instances = filter_instances_by_date(instances, start_date, end_date)
+
+        if len(instances.data) == 0:
+            raise NoReportsFoundError()
+
+        for instance in instances.data:
+            segments = client.list_report_segments(instance.id)
+            with tempfile.TemporaryDirectory() as temp_dir:
+                files = []
+                for segment in segments.data:
+                    payload = requests.get(segment.attributes.url, stream=True)
+                    payload.raise_for_status()
+
+                    csv_path = os.path.join(
+                        temp_dir, f"{segment.attributes.checksum}.csv"
+                    )
+                    with open(csv_path, "wb") as f:
+                        for chunk in payload.iter_content(chunk_size=8192):
+                            f.write(chunk)
+                    files.append(csv_path)
+                for file in files:
+                    with gzip.open(file, "rt") as f:
+                        # TODO: infer delimiter from the file itself
+                        delimiter = (
+                            "," if report_name == "App Crashes Expanded" else "\t"
+                        )
+                        reader = csv.DictReader(f, delimiter=delimiter)
+                        for row in reader:
+                            yield {
+                                "processing_date": instance.attributes.processingDate,
+                                **row,
+                            }

ingestr-0.12.8/ingestr/src/appstore/client.py

@@ -0,0 +1,126 @@
+import abc
+import time
+from typing import Optional
+
+import jwt
+import requests
+from requests.models import PreparedRequest
+
+from .models import (
+    AnalyticsReportInstancesResponse,
+    AnalyticsReportRequestsResponse,
+    AnalyticsReportResponse,
+    AnalyticsReportSegmentsResponse,
+)
+
+
+class AppStoreConnectClientInterface(abc.ABC):
+    @abc.abstractmethod
+    def list_analytics_report_requests(self, app_id) -> AnalyticsReportRequestsResponse:
+        pass
+
+    @abc.abstractmethod
+    def list_analytics_reports(
+        self, req_id: str, report_name: str
+    ) -> AnalyticsReportResponse:
+        pass
+
+    @abc.abstractmethod
+    def list_report_instances(
+        self,
+        report_id: str,
+        granularity: str = "DAILY",
+    ) -> AnalyticsReportInstancesResponse:
+        pass
+
+    @abc.abstractmethod
+    def list_report_segments(self, instance_id: str) -> AnalyticsReportSegmentsResponse:
+        pass
+
+
+class AppStoreConnectClient(AppStoreConnectClientInterface):
+    def __init__(self, key: bytes, key_id: str, issuer_id: str):
+        self.__key = key
+        self.__key_id = key_id
+        self.__issuer_id = issuer_id
+
+    def list_analytics_report_requests(self, app_id) -> AnalyticsReportRequestsResponse:
+        res = requests.get(
+            f"https://api.appstoreconnect.apple.com/v1/apps/{app_id}/analyticsReportRequests",
+            auth=self.auth,
+        )
+        res.raise_for_status()
+
+        return AnalyticsReportRequestsResponse.from_json(res.text)  # type: ignore
+
+    def list_analytics_reports(
+        self, req_id: str, report_name: str
+    ) -> AnalyticsReportResponse:
+        params = {"filter[name]": report_name}
+        res = requests.get(
+            f"https://api.appstoreconnect.apple.com/v1/analyticsReportRequests/{req_id}/reports",
+            auth=self.auth,
+            params=params,
+        )
+        res.raise_for_status()
+        return AnalyticsReportResponse.from_json(res.text)  # type: ignore
+
+    def list_report_instances(
+        self,
+        report_id: str,
+        granularity: str = "DAILY",
+    ) -> AnalyticsReportInstancesResponse:
+        data = []
+        url = f"https://api.appstoreconnect.apple.com/v1/analyticsReports/{report_id}/instances"
+        params: Optional[dict] = {"filter[granularity]": granularity}
+
+        while url:
+            res = requests.get(url, auth=self.auth, params=params)
+            res.raise_for_status()
+
+            response_data = AnalyticsReportInstancesResponse.from_json(res.text)  # type: ignore
+            data.extend(response_data.data)
+
+            url = response_data.links.next
+            params = None  # Clear params for subsequent requests
+
+        return AnalyticsReportInstancesResponse(
+            data=data,
+            links=response_data.links,
+            meta=response_data.meta,
+        )
+
+    def list_report_segments(self, instance_id: str) -> AnalyticsReportSegmentsResponse:
+        segments = []
+        url = f"https://api.appstoreconnect.apple.com/v1/analyticsReportInstances/{instance_id}/segments"
+
+        while url:
+            res = requests.get(url, auth=self.auth)
+            res.raise_for_status()
+
+            response_data = AnalyticsReportSegmentsResponse.from_json(res.text)  # type: ignore
+            segments.extend(response_data.data)
+
+            url = response_data.links.next
+
+        return AnalyticsReportSegmentsResponse(
+            data=segments, links=response_data.links, meta=response_data.meta
+        )
+
+    def auth(self, req: PreparedRequest) -> PreparedRequest:
+        headers = {
+            "alg": "ES256",
+            "kid": self.__key_id,
+        }
+        payload = {
+            "iss": self.__issuer_id,
+            "exp": int(time.time()) + 600,
+            "aud": "appstoreconnect-v1",
+        }
+        req.headers["Authorization"] = jwt.encode(
+            payload,
+            self.__key,
+            algorithm="ES256",
+            headers=headers,
+        )
+        return req

ingestr-0.12.8/ingestr/src/appstore/errors.py

@@ -0,0 +1,15 @@
+class NoReportsFoundError(Exception):
+    def __init__(self):
+        super().__init__("No Report instances found for the given date range")
+
+
+class NoOngoingReportRequestsFoundError(Exception):
+    def __init__(self):
+        super().__init__(
+            "No ONGOING report requests found (or they're stopped due to inactivity)"
+        )
+
+
+class NoSuchReportError(Exception):
+    def __init__(self, report_name):
+        super().__init__(f"No such report found: {report_name}")