ingestr 0.7.8__tar.gz → 0.8.1__tar.gz

{ingestr-0.7.8 → ingestr-0.8.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.7.8
+Version: 0.8.1
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -44,6 +44,7 @@ Requires-Dist: sqlalchemy==1.4.52
 Requires-Dist: stripe==10.7.0
 Requires-Dist: tqdm==4.66.2
 Requires-Dist: typer==0.12.3
+Requires-Dist: types-requests==2.32.0.20240907
 Description-Content-Type: text/markdown
 
 <div align="center">
@@ -178,10 +179,18 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
     <tr>
         <td colspan="3" style='text-align:center;'><strong>Platforms</strong></td>
     </tr>
+    <td>Adjust</td>
+        <td>✅</td>
+        <td>-</td>
     <tr>
         <td>Airtable</td>
         <td>✅</td>
         <td>-</td>
+    </tr>
+     <tr>
+        <td>AppsFlyer</td>
+        <td>✅</td>
+        <td>-</td>
     </tr>
     <tr>
         <td>Chess.com</td>

{ingestr-0.7.8 → ingestr-0.8.1}/README.md

@@ -130,10 +130,18 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
     <tr>
         <td colspan="3" style='text-align:center;'><strong>Platforms</strong></td>
     </tr>
+    <td>Adjust</td>
+        <td>✅</td>
+        <td>-</td>
     <tr>
         <td>Airtable</td>
         <td>✅</td>
         <td>-</td>
+    </tr>
+     <tr>
+        <td>AppsFlyer</td>
+        <td>✅</td>
+        <td>-</td>
     </tr>
     <tr>
         <td>Chess.com</td>

{ingestr-0.7.8 → ingestr-0.8.1}/docs/.vitepress/config.mjs

@@ -84,7 +84,9 @@ export default defineConfig({
             text: "Platforms",
             collapsed: false,
             items: [
+              { text: "Adjust", link: "/supported-sources/adjust.md" },
               { text: "Airtable", link: "/supported-sources/airtable.md" },
+              { text: "AppsFlyer", link: "/supported-sources/appsflyer.md" },
               { text: "Chess.com", link: "/supported-sources/chess.md" },
               {
                 text: "Facebook Ads",

ingestr-0.8.1/docs/supported-sources/adjust.md

@@ -0,0 +1,30 @@
+# Adjust
+
+[Adjust](https://www.adjust.com/) is a mobile marketing analytics platform that provides solutions for measuring and optimizing campaigns, as well as protecting user data.
+
+ingestr supports Adjust as a source.
+
+## URI Format
+
+The URI format for Adjust is as follows:
+
+```plaintext
+adjust://?api_key=<api-key-here>
+```
+
+An API token is required to retrieve reports from the Adjust reporting API. please follow the guide to [obtain a API key](https://dev.adjust.com/en/api/rs-api/authentication/).
+
+Once you complete the guide, you should have an API key. Let's say your API key is `nr_123`, here's a sample command that will copy the data from Adjust into a duckdb database:
+
+```sh
+ingestr ingest --source-uri 'adjust://?api_key=nr_123' --source-table 'campaigns' --dest-uri duckdb:///adjust.duckdb --dest-table 'adjust.output' --interval-start '2024-09-05' --interval-end '2024-09-08'
+```
+
+The result of this command will be a table in the `adjust.duckdb` database
+
+Available Source Table:
+Adjust source allows ingesting the following source into separate tables:
+
+-`Campaigns`: Retrieves data for a campaign, showing the app's revenue and network costs over multiple days.
+
+--`Creatives`: Retrieves data for a creative assest, detailing the app's revenue and network costs across multiple days

ingestr-0.8.1/docs/supported-sources/appsflyer.md

@@ -0,0 +1,28 @@
+# AppsFlyer
+
+[AppsFlyer](https://www.appsflyer.com/) is a mobile marketing analytics and attribution platform that helps businesses track, measure, and optimize their app marketing efforts across various channels.
+
+ingestr supports AppsFlyer as a source.
+
+The URI format for AppsFlyer is as follows:
+
+```plaintext
+appsflyer://?api_key=<api-key>
+```
+
+An API token is required to retrieve reports from the AppsFlyer API. Please follow the guide to [obtain a API key](https://support.appsflyer.com/hc/en-us/articles/360004562377-Managing-AppsFlyer-tokens)
+
+Once you complete the guide, you should have an API key. Let's say your API key is ey123, here's a sample command that will copy the data from AppsFlyer into a duckdb database:
+
+ingestr ingest --source-uri 'appsflyer://?api_key=ey123' --source-table 'campaigns' --dest-uri duckdb:///appsflyer.duckdb --dest-table 'appsflyer.output' --interval-start '2024-08-01' --interval-end '2024-08-28'
+
+The result of this command will be a table in the appsflyer.duckdb database
+
+Available Source Table:
+AppsFlyer source allows ingesting the following source into separate tables:
+
+-Campaigns: Retrieves data for campaigns, detailing the app's costs, loyal users, total installs, and revenue over multiple days.
+
+-Creatives: Retrieves data for a creative asset, including revenue and cost.
+
+Use these as `--source-table` parameter in the `ingestr ingest` command.

{ingestr-0.7.8 → ingestr-0.8.1}/docs/supported-sources/shopify.md

@@ -32,6 +32,12 @@ The result of this command will be a table in the `shopify.duckdb` database with
 Shopify source allows ingesting the following sources into separate tables:
 - `orders`
 - `customers`
+- `discounts`: Uses the GraphQL API, `discountNodes` query, take a look at the [Shopify docs](https://shopify.dev/docs/api/admin-graphql/2024-07/queries/discountNodes) for more details.
 - `products`
+- `inventory_items`
+- `transactions`
+- `balance`
+- `events`
+- `price_rules`: this is a deprecated table, please use `discounts` instead.
 
 Use these as `--source-table` parameter in the `ingestr ingest` command.

{ingestr-0.7.8 → ingestr-0.8.1}/ingestr/main.py

@@ -259,6 +259,7 @@ def ingest(
         },
     )
 
+    dlt.config["data_writer.buffer_max_items"] = page_size
     dlt.config["data_writer.file_max_items"] = loader_file_size
     dlt.config["extract.workers"] = extract_parallelism
     dlt.config["extract.max_parallel_items"] = extract_parallelism

ingestr-0.8.1/ingestr/src/adjust/_init_.py

@@ -0,0 +1,31 @@
+from typing import Sequence
+
+import dlt
+from dlt.sources import DltResource
+
+from .helpers import DEFAULT_DIMENSIONS, AdjustAPI
+
+
+@dlt.source(max_table_nesting=0)
+def adjust_source(
+    start_date: str,
+    end_date: str,
+    api_key: str,
+) -> Sequence[DltResource]:
+    @dlt.resource(write_disposition="merge", merge_key="day")
+    def campaigns():
+        adjust_api = AdjustAPI(api_key=api_key)
+        yield from adjust_api.fetch_report_data(
+            start_date=start_date,
+            end_date=end_date,
+        )
+
+    @dlt.resource(write_disposition="merge", merge_key="day")
+    def creatives():
+        dimensions = DEFAULT_DIMENSIONS + ["adgroup", "creative"]
+        adjust_api = AdjustAPI(api_key=api_key)
+        yield from adjust_api.fetch_report_data(
+            start_date=start_date, end_date=end_date, dimensions=dimensions
+        )
+
+    return campaigns, creatives

ingestr-0.8.1/ingestr/src/adjust/helpers.py

@@ -0,0 +1,82 @@
+import requests
+from dlt.sources.helpers.requests import Client
+from requests.exceptions import HTTPError
+
+DEFAULT_DIMENSIONS = ["campaign", "day", "app", "store_type", "channel", "country"]
+
+DEFAULT_METRICS = [
+    "network_cost",
+    "all_revenue_total_d0",
+    "ad_revenue_total_d0",
+    "revenue_total_d0",
+    "all_revenue_total_d1",
+    "ad_revenue_total_d1",
+    "revenue_total_d1",
+    "all_revenue_total_d3",
+    "ad_revenue_total_d3",
+    "revenue_total_d3",
+    "all_revenue_total_d7",
+    "ad_revenue_total_d7",
+    "revenue_total_d7",
+    "all_revenue_total_d14",
+    "ad_revenue_total_d14",
+    "revenue_total_d14",
+    "all_revenue_total_d21",
+]
+
+
+class AdjustAPI:
+    def __init__(self, api_key):
+        self.api_key = api_key
+        self.uri = "https://automate.adjust.com/reports-service/report"
+
+    def fetch_report_data(
+        self,
+        start_date,
+        end_date,
+        dimensions=DEFAULT_DIMENSIONS,
+        metrics=DEFAULT_METRICS,
+        utc_offset="+00:00",
+        ad_spend_mode="network",
+        attribution_source="first",
+        attribution_type="all",
+        cohort_maturity="immature",
+        reattributed="all",
+        sandbox="false",
+    ):
+        headers = {"Authorization": f"Bearer {self.api_key}"}
+        comma_separated_dimensions = ",".join(dimensions)
+        comma_separated_metrics = ",".join(metrics)
+        params = {
+            "date_period": f"{start_date}:{end_date}",
+            "dimensions": comma_separated_dimensions,
+            "metrics": comma_separated_metrics,
+            "utc_offset": utc_offset,
+            "ad_spend_mode": ad_spend_mode,
+            "attribution_source": attribution_source,
+            "attribution_type": attribution_type,
+            "cohort_maturity": cohort_maturity,
+            "reattributed": reattributed,
+            "sandbox": sandbox,
+        }
+
+        def retry_on_limit(
+            response: requests.Response, exception: BaseException
+        ) -> bool:
+            return response.status_code == 429
+
+        request_client = Client(
+            request_timeout=8.0,
+            raise_for_status=False,
+            retry_condition=retry_on_limit,
+            request_max_attempts=12,
+            request_backoff_factor=2,
+        ).session
+
+        response = request_client.get(self.uri, headers=headers, params=params)
+        if response.status_code == 200:
+            result = response.json()
+            items = result.get("rows", [])
+            yield items
+        else:
+            raise HTTPError(f"Request failed with status code: {response.status_code}")

ingestr-0.8.1/ingestr/src/appsflyer/_init_.py

@@ -0,0 +1,24 @@
+from typing import Iterable
+
+import dlt
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from ingestr.src.appsflyer.client import AppsflyerClient
+
+
+@dlt.source(max_table_nesting=0)
+def appsflyer_source(
+    api_key: str, start_date: str, end_date: str
+) -> Iterable[DltResource]:
+    client = AppsflyerClient(api_key)
+
+    @dlt.resource(write_disposition="merge", merge_key="install_time")
+    def campaigns() -> Iterable[TDataItem]:
+        yield from client.fetch_campaigns(start_date, end_date)
+
+    @dlt.resource(write_disposition="merge", merge_key="install_time")
+    def creatives() -> Iterable[TDataItem]:
+        yield from client.fetch_creatives(start_date, end_date)
+
+    return campaigns, creatives

ingestr-0.8.1/ingestr/src/appsflyer/client.py

@@ -0,0 +1,106 @@
+from typing import Optional
+
+import requests
+from dlt.sources.helpers.requests import Client
+from requests.exceptions import HTTPError
+
+DEFAULT_GROUPING = ["c", "geo", "app_id", "install_time"]
+DEFAULT_KPIS = [
+    "impressions",
+    "clicks",
+    "installs",
+    "cost",
+    "revenue",
+    "average_ecpi",
+    "loyal_users",
+    "uninstalls",
+    "roi",
+]
+
+
+class AppsflyerClient:
+    def __init__(self, api_key: str):
+        self.api_key = api_key
+        self.uri = "https://hq1.appsflyer.com/api/master-agg-data/v4/app/all"
+
+    def __get_headers(self):
+        return {
+            "Authorization": f"{self.api_key}",
+            "accept": "text/json",
+        }
+
+    def _fetch_data(
+        self,
+        from_date: str,
+        to_date: str,
+        maximum_rows=1000000,
+        dimensions=DEFAULT_GROUPING,
+        metrics=DEFAULT_KPIS,
+    ):
+        params = {
+            "from": from_date,
+            "to": to_date,
+            "groupings": ",".join(dimensions),
+            "kpis": ",".join(metrics),
+            "format": "json",
+            "maximum_rows": maximum_rows,
+        }
+
+        def retry_on_limit(
+            response: Optional[requests.Response], exception: Optional[BaseException]
+        ) -> bool:
+            return (
+                isinstance(response, requests.Response) and response.status_code == 429
+            )
+
+        request_client = Client(
+            request_timeout=10.0,
+            raise_for_status=False,
+            retry_condition=retry_on_limit,
+            request_max_attempts=12,
+            request_backoff_factor=2,
+        ).session
+
+        try:
+            response = request_client.get(
+                url=self.uri, headers=self.__get_headers(), params=params
+            )
+
+            if response.status_code == 200:
+                result = response.json()
+                yield result
+            else:
+                raise HTTPError(
+                    f"Request failed with status code: {response.status_code}"
+                )
+
+        except requests.RequestException as e:
+            raise HTTPError(f"Request failed: {e}")
+
+    def fetch_campaigns(
+        self,
+        start_date: str,
+        end_date: str,
+    ):
+        metrics = DEFAULT_KPIS + [
+            "cohort_day_1_revenue_per_user",
+            "cohort_day_1_total_revenue_per_user",
+            "cohort_day_3_revenue_per_user",
+            "cohort_day_3_total_revenue_per_user",
+            "cohort_day_7_total_revenue_per_user",
+            "cohort_day_7_revenue_per_user",
+            "cohort_day_14_total_revenue_per_user",
+            "cohort_day_14_revenue_per_user",
+            "cohort_day_21_total_revenue_per_user",
+            "cohort_day_21_revenue_per_user",
+            "retention_day_7",
+        ]
+        return self._fetch_data(start_date, end_date, metrics=metrics)
+
+    def fetch_creatives(
+        self,
+        start_date: str,
+        end_date: str,
+    ):
+        dimensions = DEFAULT_GROUPING + ["af_adset_id", "af_adset", "af_ad_id"]
+        return self._fetch_data(start_date, end_date, dimensions=dimensions)

{ingestr-0.7.8 → ingestr-0.8.1}/ingestr/src/factory.py

@@ -15,7 +15,9 @@ from ingestr.src.destinations import (
     SynapseDestination,
 )
 from ingestr.src.sources import (
+    AdjustSource,
     AirtableSource,
+    AppsflyerSource,
     ChessSource,
     FacebookAdsSource,
     GoogleSheetsSource,
@@ -124,8 +126,12 @@ class SourceDestinationFactory:
             return AirtableSource()
         elif self.source_scheme == "klaviyo":
             return KlaviyoSource()
+        elif self.source_scheme == "appsflyer":
+            return AppsflyerSource()
         elif self.source_scheme == "kafka":
             return KafkaSource()
+        elif self.source_scheme == "adjust":
+            return AdjustSource()
         else:
             raise ValueError(f"Unsupported source scheme: {self.source_scheme}")