ingestr 0.13.11__tar.gz → 0.13.13__tar.gz

{ingestr-0.13.11 → ingestr-0.13.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.11
+Version: 0.13.13
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -57,6 +57,7 @@ Requires-Dist: stripe==10.7.0
 Requires-Dist: tqdm==4.67.1
 Requires-Dist: typer==0.13.1
 Requires-Dist: types-requests==2.32.0.20240907
+Requires-Dist: zstd==1.5.6.1
 Provides-Extra: odbc
 Requires-Dist: pyodbc==5.1.0; extra == 'odbc'
 Provides-Extra: oracle
@@ -149,7 +150,7 @@ Pull requests are welcome. However, please open an issue first to discuss what y
     </tr>
     <tr>
         <td>ClickHouse</td>
-        <td>❌</td>
+        <td>✅</td>
         <td>✅</td>
     </tr>
     <tr>

{ingestr-0.13.11 → ingestr-0.13.13}/README.md

@@ -84,7 +84,7 @@ Pull requests are welcome. However, please open an issue first to discuss what y
     </tr>
     <tr>
         <td>ClickHouse</td>
-        <td>❌</td>
+        <td>✅</td>
         <td>✅</td>
     </tr>
     <tr>

{ingestr-0.13.11 → ingestr-0.13.13}/docs/supported-sources/applovin_max.md

@@ -6,12 +6,24 @@
 ## URI Format
 The URI format for Applovin Max is as follows:
 ```
-applovinmax://?api_key=<your_api_key>&application=<application_name>
+applovinmax://?api_key=<your_api_key>
 ```
 
 URI Parameters:
 - `api_key`: It is the `report key` which is used for authenticating the request.
-- `application`: The application package name (for Android and Fire OS) or bundle ID (for iOS).
+
+## Source Table Format
+The source table format should be specified as:
+```
+<table_name>:<application_ids>
+```
+
+Example:
+```
+user_ad_revenue:com.example.app1,com.example.app2
+```
+
+- `application_id` should be the application package name (for Android and Fire OS) or bundle ID (for iOS).
 
 ## Setting up Applovin Integration
 
@@ -19,12 +31,12 @@ URI Parameters:
 You can generate a report key from your [AppLovin dashboard](https://developers.applovin.com/en/max/max-dashboard/account/account-info/#keys).
 
 ### Example:
-To retrieve `ad revenue` data for an application `com.example.app` with api key `key_123`, and to store it in a DuckDB database, use the following command:
+To retrieve `user ad revenue` data for an application `com.example.app` with api key `key_123`, and to store it in a DuckDB database, use the following command:
 
 ```sh
 ingestr ingest \
-    --source-uri "applovinmax://?api_key=key_123&application=com.example.app" \
-    --source-table "ad_revenue" \
+    --source-uri "applovinmax://?api_key=key_123" \
+    --source-table "user_ad_revenue:com.example.app" \
     --dest-uri "duckdb:///applovin_max.db"  \
     --dest-table "dest.ad_revenue"
 ```
@@ -33,4 +45,4 @@ By default, `ingestr` retrieves data for the last 30 days. For a custom date ran
 <img alt="applovin_max_img" src="../media/applovin_max.png"/>
 
 ## Table
-[ad_revenue](https://developers.applovin.com/en/max/reporting-apis/user-level-ad-revenue-api/): Provides daily metrics from the user-level ad revenue API.User-level revenue data is available eight hours after UTC day end. So, for example, data for UTC 2025-01-01 is available on UTC 2025-01-02 after 08:00.
+[user_ad_revenue](https://developers.applovin.com/en/max/reporting-apis/user-level-ad-revenue-api/): Provides daily metrics from the user level ad revenue API.User-level revenue data is available eight hours after UTC day end. So, for example, data for UTC 2025-01-01 is available on UTC 2025-01-02 after 08:00.

ingestr-0.13.13/ingestr/src/applovin_max/__init__.py

@@ -0,0 +1,115 @@
+from datetime import timedelta
+from typing import Iterator
+
+import dlt
+import pandas as pd  # type: ignore[import-untyped]
+import pendulum
+import requests
+from dlt.sources import DltResource
+from dlt.sources.helpers.requests import Client
+from pendulum.date import Date
+
+
+@dlt.source(max_table_nesting=0)
+def applovin_max_source(
+    start_date: Date,
+    applications: list[str],
+    api_key: str,
+    end_date: Date | None,
+) -> DltResource:
+    @dlt.resource(
+        name="user_ad_revenue",
+        write_disposition="merge",
+        merge_key="partition_date",
+        columns={
+            "partition_date": {"data_type": "date", "partition": True},
+        },
+    )
+    def fetch_ad_revenue_report(
+        dateTime=(
+            dlt.sources.incremental(
+                "partition_date",
+                initial_value=start_date,
+                end_value=end_date,
+                range_start="closed",
+                range_end="closed",
+            )
+        ),
+    ) -> Iterator[dict]:
+        url = "https://r.applovin.com/max/userAdRevenueReport"
+        start_date = dateTime.last_value
+
+        if dateTime.end_value is None:
+            end_date = (pendulum.yesterday("UTC")).date()
+        else:
+            end_date = dateTime.end_value
+
+        client = create_client()
+        platforms = ["ios", "android", "fireos"]
+
+        for app in applications:
+            current_date = start_date
+            while current_date <= end_date:
+                for platform in platforms:
+                    df = get_data(
+                        url=url,
+                        current_date=current_date,
+                        application=app,
+                        api_key=api_key,
+                        client=client,
+                        platform=platform,
+                    )
+                    if df is not None:
+                        yield df
+                current_date = current_date + timedelta(days=1)
+
+    return fetch_ad_revenue_report
+
+
+def create_client() -> requests.Session:
+    return Client(
+        request_timeout=10.0,
+        raise_for_status=False,
+        retry_condition=retry_on_limit,
+        request_max_attempts=12,
+    ).session
+
+
+def retry_on_limit(
+    response: requests.Response | None, exception: BaseException | None
+) -> bool:
+    if response is None:
+        return False
+    return response.status_code == 429
+
+
+def get_data(
+    url: str,
+    current_date: Date,
+    application: str,
+    api_key: str,
+    platform: str,
+    client: requests.Session,
+):
+    params = {
+        "api_key": api_key,
+        "date": current_date.isoformat(),
+        "platform": platform,
+        "application": application,
+        "aggregated": "false",
+    }
+
+    response = client.get(url=url, params=params)
+
+    if response.status_code != 200:
+        if response.status_code == 404:
+            if "No Mediation App Id found for platform" in response.text:
+                return None
+        error_message = f"AppLovin MAX API error (status {response.status_code}): {response.text}"
+        raise requests.HTTPError(error_message)
+    
+    response_url = response.json().get("ad_revenue_report_url")
+    df = pd.read_csv(response_url)
+    df["Date"] = pd.to_datetime(df["Date"])
+    df["partition_date"] = df["Date"].dt.date
+    return df

ingestr-0.13.13/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.13"

{ingestr-0.13.11 → ingestr-0.13.13}/ingestr/src/personio/__init__.py

@@ -165,7 +165,7 @@ def personio_source(
         Returns:
             Iterable: A generator of attendances.
         """
-        
+
         end_date = end_date or pendulum.now()
         if updated_at.last_value:
             updated_iso = updated_at.last_value.format("YYYY-MM-DDTHH:mm:ss")

{ingestr-0.13.11 → ingestr-0.13.13}/ingestr/src/personio/helpers.py

@@ -1,4 +1,5 @@
 """Personio source helpers"""
+
 from typing import Any, Iterable, Optional
 from urllib.parse import urljoin
 

{ingestr-0.13.11 → ingestr-0.13.13}/ingestr/src/sources.py

@@ -402,6 +402,7 @@ class LocalCsvSource:
                         if inc_value < incremental.start_value:
                             continue
 
+                    dictionary = self.remove_empty_columns(dictionary)
                     page.append(dictionary)
                     current_items += 1
                 else:
@@ -425,6 +426,9 @@ class LocalCsvSource:
             )
         )
 
+    def remove_empty_columns(self, row: Dict[str, str]) -> Dict[str, str]:
+        return {k: v for k, v in row.items() if v.strip() != ""}
+
 
 class NotionSource:
     table_builder: Callable
@@ -1829,6 +1833,9 @@ class AppLovinSource:
 
 
 class ApplovinMaxSource:
+    #expected uri format: applovinmax://?api_key=<api_key>
+    #expected table format: user_ad_revenue:app_id_1,app_id_2
+
     def handles_incrementality(self) -> bool:
         return True
 
@@ -1839,38 +1846,54 @@ class ApplovinMaxSource:
         api_key = params.get("api_key")
         if api_key is None:
             raise ValueError("api_key is required to connect to AppLovin Max API.")
+        
+        AVAILABLE_TABLES = ["user_ad_revenue"]
 
-        application = params.get("application")
-        if application is None:
-            raise ValueError("application is required to connect to AppLovin Max API.")
-
-        interval_start = kwargs.get("interval_start")
-        interval_end = kwargs.get("interval_end")
+        table_fields = table.split(":")
+        requested_table = table_fields[0]
 
-        if "ad_revenue" in table:
-            table = "ad_revenue"
-        else:
+        if len(table_fields) != 2:
+            raise ValueError(
+                "Invalid table format. Expected format is user_ad_revenue:app_id_1,app_id_2"
+            )
+        
+        if requested_table not in AVAILABLE_TABLES:
+            raise ValueError(
+                f"Table name '{requested_table}' is not supported for AppLovin Max source yet."
+                f"Only '{AVAILABLE_TABLES}' are currently supported. "
+                "If you need additional tables, please create a GitHub issue at "
+                "https://github.com/bruin-data/ingestr"
+            )
+        
+        applications = [i for i in table_fields[1].replace(" ", "").split(",") if i.strip()]
+        if len(applications) == 0:
             raise ValueError(
-                f"Table name '{table}' is not supported for AppLovin Max source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+                "At least one application id is required"
             )
+    
+        if len(applications) != len(set(applications)):
+            raise ValueError(
+                "Application ids must be unique."
+            )
+
+        interval_start = kwargs.get("interval_start")
+        interval_end = kwargs.get("interval_end")
 
         now = pendulum.now("UTC")
         default_start = now.subtract(days=30).date()
 
         start_date = (
-            interval_start if interval_start is not None else default_start
-        ).strftime("%Y-%m-%d")
-
-        end_date = (
-            interval_end.strftime("%Y-%m-%d") if interval_end is not None else None
+            interval_start.date() if interval_start is not None else default_start
         )
 
+        end_date = interval_end.date() if interval_end is not None else None
+
         return applovin_max_source(
             start_date=start_date,
             end_date=end_date,
             api_key=api_key[0],
-            application=application[0],
-        ).with_resources(table)
+            applications=applications,
+        ).with_resources(requested_table)
 
 
 class SalesforceSource:

{ingestr-0.13.11 → ingestr-0.13.13}/requirements.txt

@@ -45,3 +45,4 @@ clickhouse-connect==0.8.14
 clickhouse-driver==0.2.9
 clickhouse-sqlalchemy==0.2.7
 simple-salesforce==1.12.6
+zstd==1.5.6.1

ingestr-0.13.11/ingestr/src/applovin_max/__init__.py

@@ -1,99 +0,0 @@
-from typing import Iterator
-
-import dlt
-import pandas as pd  # type: ignore[import-untyped]
-import pendulum
-import requests
-from dlt.sources import DltResource
-from dlt.sources.helpers.requests import Client
-from pendulum.date import Date
-
-
-@dlt.source(max_table_nesting=0)
-def applovin_max_source(
-    start_date: str,
-    application: str,
-    api_key: str,
-    end_date: str | None,
-) -> DltResource:
-    @dlt.resource(
-        name="ad_revenue",
-        write_disposition="merge",
-        merge_key="_partition_date",
-    )
-    def fetch_ad_revenue_report(
-        dateTime=(
-            dlt.sources.incremental(
-                "_partition_date",
-                initial_value=start_date,
-                end_value=end_date,
-                range_start="closed",
-                range_end="closed",
-            )
-        ),
-    ) -> Iterator[dict]:
-        url = "https://r.applovin.com/max/userAdRevenueReport"
-        start_date = pendulum.from_format(dateTime.last_value, "YYYY-MM-DD").date()
-        if dateTime.end_value is None:
-            end_date = (pendulum.yesterday("UTC")).date()
-        else:
-            end_date = pendulum.from_format(dateTime.end_value, "YYYY-MM-DD").date()
-        yield get_data(
-            url=url,
-            start_date=start_date,
-            end_date=end_date,
-            application=application,
-            api_key=api_key,
-        )
-
-    return fetch_ad_revenue_report
-
-
-def create_client() -> requests.Session:
-    return Client(
-        request_timeout=10.0,
-        raise_for_status=False,
-        retry_condition=retry_on_limit,
-        request_max_attempts=12,
-    ).session
-
-
-def retry_on_limit(
-    response: requests.Response | None, exception: BaseException | None
-) -> bool:
-    if response is None:
-        return False
-    return response.status_code == 429
-
-
-def get_data(
-    url: str, start_date: Date, end_date: Date, application: str, api_key: str
-):
-    client = create_client()
-    platforms = ["ios", "android", "fireos"]
-    current_date = start_date
-    while current_date <= end_date:
-        for platform in platforms:
-            params = {
-                "api_key": api_key,
-                "date": current_date.strftime("%Y-%m-%d"),
-                "platform": platform,
-                "application": application,
-                "aggregated": "false",
-            }
-
-            response = client.get(url=url, params=params)
-
-            if response.status_code == 400:
-                raise ValueError(response.text)
-
-            if response.status_code != 200:
-                continue
-
-            response_url = response.json().get("ad_revenue_report_url")
-            df = pd.read_csv(response_url)
-            df["Date"] = pd.to_datetime(df["Date"])
-            df["_partition_date"] = df["Date"].dt.strftime("%Y-%m-%d")
-            yield df
-
-        current_date = current_date.add(days=1)

ingestr-0.13.11/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.11"