ingestr 0.13.11__py3-none-any.whl → 0.13.13__py3-none-any.whl

ingestr/src/applovin_max/__init__.py

@@ -1,3 +1,4 @@
+from datetime import timedelta
 from typing import Iterator
 
 import dlt
@@ -11,20 +12,23 @@ from pendulum.date import Date
 
 @dlt.source(max_table_nesting=0)
 def applovin_max_source(
-    start_date: str,
-    application: str,
+    start_date: Date,
+    applications: list[str],
     api_key: str,
-    end_date: str | None,
+    end_date: Date | None,
 ) -> DltResource:
     @dlt.resource(
-        name="ad_revenue",
+        name="user_ad_revenue",
         write_disposition="merge",
-        merge_key="_partition_date",
+        merge_key="partition_date",
+        columns={
+            "partition_date": {"data_type": "date", "partition": True},
+        },
     )
     def fetch_ad_revenue_report(
         dateTime=(
             dlt.sources.incremental(
-                "_partition_date",
+                "partition_date",
                 initial_value=start_date,
                 end_value=end_date,
                 range_start="closed",
@@ -33,18 +37,31 @@ def applovin_max_source(
         ),
     ) -> Iterator[dict]:
         url = "https://r.applovin.com/max/userAdRevenueReport"
-        start_date = pendulum.from_format(dateTime.last_value, "YYYY-MM-DD").date()
+        start_date = dateTime.last_value
+
         if dateTime.end_value is None:
             end_date = (pendulum.yesterday("UTC")).date()
         else:
-            end_date = pendulum.from_format(dateTime.end_value, "YYYY-MM-DD").date()
-        yield get_data(
-            url=url,
-            start_date=start_date,
-            end_date=end_date,
-            application=application,
-            api_key=api_key,
-        )
+            end_date = dateTime.end_value
+
+        client = create_client()
+        platforms = ["ios", "android", "fireos"]
+
+        for app in applications:
+            current_date = start_date
+            while current_date <= end_date:
+                for platform in platforms:
+                    df = get_data(
+                        url=url,
+                        current_date=current_date,
+                        application=app,
+                        api_key=api_key,
+                        client=client,
+                        platform=platform,
+                    )
+                    if df is not None:
+                        yield df
+                current_date = current_date + timedelta(days=1)
 
     return fetch_ad_revenue_report
 
@@ -67,33 +84,32 @@ def retry_on_limit(
 
 
 def get_data(
-    url: str, start_date: Date, end_date: Date, application: str, api_key: str
+    url: str,
+    current_date: Date,
+    application: str,
+    api_key: str,
+    platform: str,
+    client: requests.Session,
 ):
-    client = create_client()
-    platforms = ["ios", "android", "fireos"]
-    current_date = start_date
-    while current_date <= end_date:
-        for platform in platforms:
-            params = {
-                "api_key": api_key,
-                "date": current_date.strftime("%Y-%m-%d"),
-                "platform": platform,
-                "application": application,
-                "aggregated": "false",
-            }
-
-            response = client.get(url=url, params=params)
-
-            if response.status_code == 400:
-                raise ValueError(response.text)
-
-            if response.status_code != 200:
-                continue
+    params = {
+        "api_key": api_key,
+        "date": current_date.isoformat(),
+        "platform": platform,
+        "application": application,
+        "aggregated": "false",
+    }
 
-            response_url = response.json().get("ad_revenue_report_url")
-            df = pd.read_csv(response_url)
-            df["Date"] = pd.to_datetime(df["Date"])
-            df["_partition_date"] = df["Date"].dt.strftime("%Y-%m-%d")
-            yield df
+    response = client.get(url=url, params=params)
 
-        current_date = current_date.add(days=1)
+    if response.status_code != 200:
+        if response.status_code == 404:
+            if "No Mediation App Id found for platform" in response.text:
+                return None
+        error_message = f"AppLovin MAX API error (status {response.status_code}): {response.text}"
+        raise requests.HTTPError(error_message)
+    
+    response_url = response.json().get("ad_revenue_report_url")
+    df = pd.read_csv(response_url)
+    df["Date"] = pd.to_datetime(df["Date"])
+    df["partition_date"] = df["Date"].dt.date
+    return df

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.11"
+version = "v0.13.13"

ingestr/src/factory.py

@@ -41,6 +41,7 @@ from ingestr.src.sources import (
     LocalCsvSource,
     MongoDbSource,
     NotionSource,
+    PersonioSource,
     S3Source,
     SalesforceSource,
     ShopifySource,
@@ -49,7 +50,6 @@ from ingestr.src.sources import (
     StripeAnalyticsSource,
     TikTokSource,
     ZendeskSource,
-    PersonioSource,
 )
 
 SQL_SOURCE_SCHEMES = [

ingestr/src/personio/__init__.py

@@ -165,7 +165,7 @@ def personio_source(
         Returns:
             Iterable: A generator of attendances.
         """
-        
+
         end_date = end_date or pendulum.now()
         if updated_at.last_value:
             updated_iso = updated_at.last_value.format("YYYY-MM-DDTHH:mm:ss")

ingestr/src/personio/helpers.py

@@ -1,4 +1,5 @@
 """Personio source helpers"""
+
 from typing import Any, Iterable, Optional
 from urllib.parse import urljoin
 

ingestr/src/sources.py

@@ -402,6 +402,7 @@ class LocalCsvSource:
                         if inc_value < incremental.start_value:
                             continue
 
+                    dictionary = self.remove_empty_columns(dictionary)
                     page.append(dictionary)
                     current_items += 1
                 else:
@@ -425,6 +426,9 @@ class LocalCsvSource:
             )
         )
 
+    def remove_empty_columns(self, row: Dict[str, str]) -> Dict[str, str]:
+        return {k: v for k, v in row.items() if v.strip() != ""}
+
 
 class NotionSource:
     table_builder: Callable
@@ -1829,6 +1833,9 @@ class AppLovinSource:
 
 
 class ApplovinMaxSource:
+    #expected uri format: applovinmax://?api_key=<api_key>
+    #expected table format: user_ad_revenue:app_id_1,app_id_2
+
     def handles_incrementality(self) -> bool:
         return True
 
@@ -1839,38 +1846,54 @@ class ApplovinMaxSource:
         api_key = params.get("api_key")
         if api_key is None:
             raise ValueError("api_key is required to connect to AppLovin Max API.")
+        
+        AVAILABLE_TABLES = ["user_ad_revenue"]
 
-        application = params.get("application")
-        if application is None:
-            raise ValueError("application is required to connect to AppLovin Max API.")
-
-        interval_start = kwargs.get("interval_start")
-        interval_end = kwargs.get("interval_end")
+        table_fields = table.split(":")
+        requested_table = table_fields[0]
 
-        if "ad_revenue" in table:
-            table = "ad_revenue"
-        else:
+        if len(table_fields) != 2:
+            raise ValueError(
+                "Invalid table format. Expected format is user_ad_revenue:app_id_1,app_id_2"
+            )
+        
+        if requested_table not in AVAILABLE_TABLES:
+            raise ValueError(
+                f"Table name '{requested_table}' is not supported for AppLovin Max source yet."
+                f"Only '{AVAILABLE_TABLES}' are currently supported. "
+                "If you need additional tables, please create a GitHub issue at "
+                "https://github.com/bruin-data/ingestr"
+            )
+        
+        applications = [i for i in table_fields[1].replace(" ", "").split(",") if i.strip()]
+        if len(applications) == 0:
             raise ValueError(
-                f"Table name '{table}' is not supported for AppLovin Max source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+                "At least one application id is required"
             )
+    
+        if len(applications) != len(set(applications)):
+            raise ValueError(
+                "Application ids must be unique."
+            )
+
+        interval_start = kwargs.get("interval_start")
+        interval_end = kwargs.get("interval_end")
 
         now = pendulum.now("UTC")
         default_start = now.subtract(days=30).date()
 
         start_date = (
-            interval_start if interval_start is not None else default_start
-        ).strftime("%Y-%m-%d")
-
-        end_date = (
-            interval_end.strftime("%Y-%m-%d") if interval_end is not None else None
+            interval_start.date() if interval_start is not None else default_start
         )
 
+        end_date = interval_end.date() if interval_end is not None else None
+
         return applovin_max_source(
             start_date=start_date,
             end_date=end_date,
             api_key=api_key[0],
-            application=application[0],
-        ).with_resources(table)
+            applications=applications,
+        ).with_resources(requested_table)
 
 
 class SalesforceSource:

{ingestr-0.13.11.dist-info → ingestr-0.13.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.11
+Version: 0.13.13
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -57,6 +57,7 @@ Requires-Dist: stripe==10.7.0
 Requires-Dist: tqdm==4.67.1
 Requires-Dist: typer==0.13.1
 Requires-Dist: types-requests==2.32.0.20240907
+Requires-Dist: zstd==1.5.6.1
 Provides-Extra: odbc
 Requires-Dist: pyodbc==5.1.0; extra == 'odbc'
 Provides-Extra: oracle
@@ -149,7 +150,7 @@ Pull requests are welcome. However, please open an issue first to discuss what y
     </tr>
     <tr>
         <td>ClickHouse</td>
-        <td>❌</td>
+        <td>✅</td>
         <td>✅</td>
     </tr>
     <tr>

{ingestr-0.13.11.dist-info → ingestr-0.13.13.dist-info}/RECORD

@@ -1,13 +1,13 @@
 ingestr/main.py,sha256=ufn8AcM2ID80ChUApJzYDjnQaurMXOkYfTm6GzAggSQ,24746
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=LtEZWoUhm5i2aKerdgEpLtNCf3fdhGGMM4td-LRZVbY,1407
-ingestr/src/buildinfo.py,sha256=PnFKBMVizeXpYaYJ6rkY9m_oU0QCJzbLAOJyEQ8gyRg,21
+ingestr/src/buildinfo.py,sha256=hLtrn_2NNb0KhD_VpjlyF4uvE7jjsD2W10llc0aDpxE,21
 ingestr/src/destinations.py,sha256=vrGij4qMPCdXTMIimROWBJFqzOqCM4DFmgyubgSHejA,11279
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=dOdY4fzeQ-2dgFBGIDFD5ilxpYNfCVqQOureuWzOL-w,5127
+ingestr/src/factory.py,sha256=lvq-RTdzCqok-F0IVCZN0Q9ALVnuhNT06T1ju3lnPn8,5127
 ingestr/src/filters.py,sha256=0JQXeAr2APFMnW2sd-6BlAMWv93bXV17j8b5MM8sHmM,580
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
-ingestr/src/sources.py,sha256=YlWokgTZoeMQ6PVb9UVU3I99R0cdhkYjEzPf5LNGs30,68582
+ingestr/src/sources.py,sha256=vCZ20jVD5jCCNyTxR7VxNXpJFCE218t0ydbaZwls48M,69510
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -15,7 +15,7 @@ ingestr/src/adjust/__init__.py,sha256=ULjtJqrNS6XDvUyGl0tjl12-tLyXlCgeFe2icTbtu3
 ingestr/src/adjust/adjust_helpers.py,sha256=av97NPSn-hQtTbAC0vUSCAWYePmOiG5R-DGdMssm7FQ,3646
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
 ingestr/src/applovin/__init__.py,sha256=X_YCLppPrnL8KXfYWICE_uDfMzHHH3JZ-DBGZ1RlaOI,6984
-ingestr/src/applovin_max/__init__.py,sha256=1NUOeJzRyZZQ95KEirbrlSrk-8SNc9JrlM_5pGgBgHg,2878
+ingestr/src/applovin_max/__init__.py,sha256=o0aL4jBZqwK528MVw9dS1G5EZbF4tx6_Ef0IfqkhAT0,3294
 ingestr/src/appsflyer/_init_.py,sha256=ne2-9FQ654Drtd3GkKQv8Bwb6LEqCnJw49MfO5Jyzgs,739
 ingestr/src/appsflyer/client.py,sha256=TNmwakLzmO6DZW3wcfLfQRl7aNBHgFqSsk4ef-MmJ1w,3084
 ingestr/src/appstore/__init__.py,sha256=3P4VZH2WJF477QjW19jMTwu6L8DXcLkYSdutnvp3AmM,4742
@@ -74,8 +74,8 @@ ingestr/src/notion/settings.py,sha256=MwQVZViJtnvOegfjXYc_pJ50oUYgSRPgwqu7TvpeMO
 ingestr/src/notion/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestr/src/notion/helpers/client.py,sha256=QXuudkf5Zzff98HRsCqA1g1EZWIrnfn1falPrnKg_y4,5500
 ingestr/src/notion/helpers/database.py,sha256=gigPibTeVefP3lA-8w4aOwX67pj7RlciPk5koDs1ry8,2737
-ingestr/src/personio/__init__.py,sha256=CQ8XX8Q8BG-wgoen3emhe_r8Cx414Fux7P8jQNawWvY,11646
-ingestr/src/personio/helpers.py,sha256=OmeMzfg4MVtpI7f75D3-9OGZb8SDsKyz0svNm1zJLTw,2900
+ingestr/src/personio/__init__.py,sha256=sHYpoV-rg-kA1YsflctChis0hKcTrL6mka9O0CHV4zA,11638
+ingestr/src/personio/helpers.py,sha256=EKmBN0Lf4R0lc3yqqs7D-RjoZ75E8gPcctt59xwHxrY,2901
 ingestr/src/salesforce/__init__.py,sha256=2hik5pRrxVODdDTlUEMoyccNC07zozjnxkMHcjMT1qA,4558
 ingestr/src/salesforce/helpers.py,sha256=QTdazBt-qRTBbCQMZnyclIaDQFmBixBy_RDKD00Lt-8,2492
 ingestr/src/shopify/__init__.py,sha256=PF_6VQnS065Br1UzSIekTVXBu3WtrMQL_v5CfbfaX5Y,63151
@@ -108,8 +108,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.11.dist-info/METADATA,sha256=8vjvshEDHgAZEMt3ykbUSlEl_Ky0KtHf6p6vjT6RDGI,9171
-ingestr-0.13.11.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.11.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.11.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.11.dist-info/RECORD,,
+ingestr-0.13.13.dist-info/METADATA,sha256=D8jPk86Pl8XTtLvx2VFSbjC-wqCz9X0r8K74q29T26o,9200
+ingestr-0.13.13.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.13.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.13.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.13.dist-info/RECORD,,