ingestr 0.12.2__py3-none-any.whl → 0.12.4__py3-none-any.whl

ingestr/src/tiktok_ads/__init__.py

@@ -1,4 +1,4 @@
-from typing import Iterable, Optional
+from typing import Iterable
 
 import dlt
 import pendulum
@@ -8,6 +8,39 @@ from dlt.sources import DltResource
 
 from .tiktok_helpers import TikTokAPI
 
+KNOWN_TYPE_HINTS = {
+    "spend": {"data_type": "decimal"},
+    "billed_cost": {"data_type": "decimal"},
+    "cash_spend": {"data_type": "decimal"},
+    "voucher_spend": {"data_type": "decimal"},
+    "cpc": {"data_type": "decimal"},
+    "cpm": {"data_type": "decimal"},
+    "impressions": {"data_type": "bigint"},
+    "gross_impressions": {"data_type": "bigint"},
+    "clicks": {"data_type": "bigint"},
+    "ctr": {"data_type": "decimal"},
+    "reach": {"data_type": "bigint"},
+    "cost_per_1000_reached": {"data_type": "decimal"},
+    "frequency": {"data_type": "decimal"},
+    "conversion": {"data_type": "bigint"},
+    "cost_per_conversion": {"data_type": "decimal"},
+    "conversion_rate": {"data_type": "decimal"},
+    "conversion_rate_v2": {"data_type": "decimal"},
+    "real_time_conversion": {"data_type": "bigint"},
+    "real_time_cost_per_conversion": {"data_type": "decimal"},
+    "real_time_conversion_rate": {"data_type": "decimal"},
+    "real_time_conversion_rate_v2": {"data_type": "decimal"},
+    "result": {"data_type": "bigint"},
+    "cost_per_result": {"data_type": "decimal"},
+    "result_rate": {"data_type": "decimal"},
+    "real_time_result": {"data_type": "bigint"},
+    "real_time_cost_per_result": {"data_type": "decimal"},
+    "real_time_result_rate": {"data_type": "decimal"},
+    "secondary_goal_result": {"data_type": "bigint"},
+    "cost_per_secondary_goal_result": {"data_type": "decimal"},
+    "secondary_goal_result_rate": {"data_type": "decimal"},
+}
+
 
 def find_intervals(
     current_date: pendulum.DateTime,
@@ -23,42 +56,27 @@ def find_intervals(
     return intervals
 
 
-def fetch_tiktok_reports(
-    tiktok_api: TikTokAPI,
-    current_date: pendulum.DateTime,
-    interval_end: pendulum.DateTime,
-    advertiser_id: str,
-    dimensions: list[str],
-    metrics: list[str],
-    filters: Optional[dict] | None,
-) -> Iterable[TDataItem]:
-    try:
-        yield from tiktok_api.fetch_pages(
-            advertiser_id=advertiser_id,
-            start_time=current_date,
-            end_time=interval_end,
-            dimensions=dimensions,
-            metrics=metrics,
-            filters=None,
-        )
-    except Exception as e:
-        raise RuntimeError(f"Error fetching TikTok report: {e}")
-
-
 @dlt.source(max_table_nesting=0)
 def tiktok_source(
     start_date: pendulum.DateTime,
     end_date: pendulum.DateTime,
     access_token: str,
-    advertiser_id: str,
-    time_zone: str,
+    advertiser_ids: list[str],
+    timezone: str,
     page_size: int,
+    filtering_param: bool,
+    filter_name: str,
+    filter_value: list[int],
     dimensions: list[str],
     metrics: list[str],
-    filters=None,
 ) -> DltResource:
     tiktok_api = TikTokAPI(
-        access_token=access_token, time_zone=time_zone, page_size=page_size
+        access_token=access_token,
+        timezone=timezone,
+        page_size=page_size,
+        filtering_param=filtering_param,
+        filter_name=filter_name,
+        filter_value=filter_value,
     )
     incremental_loading_param = ""
     is_incremental = False
@@ -74,17 +92,34 @@ def tiktok_source(
         is_incremental = True
         interval_days = 0
 
-    @dlt.resource(write_disposition="merge", primary_key=dimensions)
+    type_hints = {
+        "advertiser_id": {"data_type": "text"},
+    }
+    for dimension in dimensions:
+        if dimension in KNOWN_TYPE_HINTS:
+            type_hints[dimension] = KNOWN_TYPE_HINTS[dimension]
+    for metric in metrics:
+        if metric in KNOWN_TYPE_HINTS:
+            type_hints[metric] = KNOWN_TYPE_HINTS[metric]
+
+    @dlt.resource(
+        write_disposition="merge",
+        primary_key=dimensions + ["advertiser_id"],
+        columns=type_hints,
+        parallelized=True,
+    )
     def custom_reports(
-        datetime=dlt.sources.incremental(incremental_loading_param, start_date)
-        if is_incremental
-        else None,
+        datetime=(
+            dlt.sources.incremental(incremental_loading_param, start_date)
+            if is_incremental
+            else None
+        ),
     ) -> Iterable[TDataItem]:
-        current_date = start_date.in_tz(time_zone)
+        current_date = start_date.in_tz(timezone)
 
         if datetime is not None:
             datetime_str = datetime.last_value
-            current_date = ensure_pendulum_datetime(datetime_str).in_tz(time_zone)
+            current_date = ensure_pendulum_datetime(datetime_str).in_tz(timezone)
 
         list_of_interval = find_intervals(
             current_date=current_date,
@@ -93,14 +128,12 @@ def tiktok_source(
         )
 
         for start, end in list_of_interval:
-            yield from fetch_tiktok_reports(
-                tiktok_api=tiktok_api,
-                current_date=start,
-                interval_end=end,
-                advertiser_id=advertiser_id,
+            yield tiktok_api.fetch_pages(
+                advertiser_ids=advertiser_ids,
+                start_time=start,
+                end_time=end,
                 dimensions=dimensions,
                 metrics=metrics,
-                filters=None,
             )
 
     return custom_reports

ingestr/src/tiktok_ads/tiktok_helpers.py

@@ -25,17 +25,17 @@ def create_client() -> requests.Session:
     ).session
 
 
-def flat_structure(items, time_zone="UTC"):
+def flat_structure(items, timezone="UTC"):
     for item in items:
         if "dimensions" in item:
             for key, value in item["dimensions"].items():
                 if key == "stat_time_day":
                     item["stat_time_day"] = ensure_pendulum_datetime(value).in_tz(
-                        time_zone
+                        timezone
                     )
                 elif key == "stat_time_hour":
                     item["stat_time_hour"] = ensure_pendulum_datetime(value).in_tz(
-                        time_zone
+                        timezone
                     )
                 else:
                     item[key] = value
@@ -49,15 +49,26 @@ def flat_structure(items, time_zone="UTC"):
 
 
 class TikTokAPI:
-    def __init__(self, access_token, time_zone, page_size):
+    def __init__(
+        self,
+        access_token,
+        timezone,
+        page_size,
+        filtering_param,
+        filter_name,
+        filter_value,
+    ):
         self.headers = {
             "Access-Token": access_token,
         }
-        self.time_zone = time_zone
+        self.timezone = timezone
         self.page_size = page_size
+        self.filtering_param = filtering_param
+        self.filter_name = filter_name
+        self.filter_value = filter_value
 
     def fetch_pages(
-        self, advertiser_id: str, start_time, end_time, dimensions, metrics, filters
+        self, advertiser_ids: list[str], start_time, end_time, dimensions, metrics
     ):
         data_level_mapping = {
             "advertiser_id": "AUCTION_ADVERTISER",
@@ -75,8 +86,15 @@ class TikTokAPI:
         start_time = ensure_pendulum_datetime(start_time).to_date_string()
         end_time = ensure_pendulum_datetime(end_time).to_date_string()
 
-        self.params = {
-            "advertiser_id": advertiser_id,
+        filtering = [
+            {
+                "field_name": self.filter_name,
+                "filter_type": "IN",
+                "filter_value": json.dumps(self.filter_value),
+            }
+        ]
+        params = {
+            "advertiser_ids": json.dumps(advertiser_ids),
             "report_type": "BASIC",
             "data_level": data_level,
             "start_date": start_time,
@@ -85,12 +103,13 @@ class TikTokAPI:
             "dimensions": json.dumps(dimensions),
             "metrics": json.dumps(metrics),
         }
+
+        if self.filtering_param:
+            params["filtering"] = json.dumps(filtering)
         client = create_client()
         while True:
-            self.params["page"] = current_page
-            response = client.get(
-                url=BASE_URL, headers=self.headers, params=self.params
-            )
+            params["page"] = current_page
+            response = client.get(url=BASE_URL, headers=self.headers, params=params)
 
             result = response.json()
             if result.get("message") != "OK":
@@ -99,7 +118,7 @@ class TikTokAPI:
             result_data = result.get("data", {})
             items = result_data.get("list", [])
 
-            flat_structure(items=items, time_zone=self.time_zone)
+            flat_structure(items=items, timezone=self.timezone)
 
             yield items
 

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.12.2"
+__version__ = "0.12.4"

{ingestr-0.12.2.dist-info → ingestr-0.12.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.12.2
+Version: 0.12.4
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -21,6 +21,7 @@ Requires-Dist: dlt==1.4.0
 Requires-Dist: duckdb-engine==0.13.5
 Requires-Dist: duckdb==1.1.3
 Requires-Dist: facebook-business==20.0.0
+Requires-Dist: google-analytics-data==0.18.15
 Requires-Dist: google-api-python-client==2.130.0
 Requires-Dist: google-cloud-bigquery-storage==2.24.0
 Requires-Dist: mysql-connector-python==9.1.0

{ingestr-0.12.2.dist-info → ingestr-0.12.4.dist-info}/RECORD

@@ -1,12 +1,12 @@
-ingestr/main.py,sha256=wkU2uLMy1q8YarJ9mXNfJepeRjp6AuPDeNDOmMUt6n0,22309
+ingestr/main.py,sha256=AG6ycOEpCyBN1qEOzW3j8sKK8KX0mrBAL-A25MdRldY,24712
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/destinations.py,sha256=zcHJIIHAZmcD9sJomd6G1Bc-1KsxnBD2aByOSV_9L3g,8850
-ingestr/src/factory.py,sha256=UyE1TzTHn_V8JZno5SSYfQsho1eFYzzvOylogw4S49E,4389
+ingestr/src/factory.py,sha256=aE7TjHzONb4DKYcfh_6-CJJfvs4lmw7iUySvSm4yQbM,4516
 ingestr/src/filters.py,sha256=0JQXeAr2APFMnW2sd-6BlAMWv93bXV17j8b5MM8sHmM,580
-ingestr/src/sources.py,sha256=QCyfkhLl5jgmosZUeh4BTrmqHk74Vus7zLgk_MBdPhc,41096
+ingestr/src/sources.py,sha256=zkK24y3jyucbrW2MU3i0Rx1SImZWatM9_A_8Wa7ExCM,51887
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
-ingestr/src/version.py,sha256=NJQQPiZZfrBXFMqZlsia0JrhloS2PexbdxYYUs0c2Us,23
+ingestr/src/version.py,sha256=DoMS9KOhsApLyuLYhLEsd5nmoLFQ_IvVkEs_jKRzFk8,23
 ingestr/src/adjust/__init__.py,sha256=NaRNwDhItG8Q7vUHw7zQvyfWjmT32M0CSc5ufjmBM9U,3067
 ingestr/src/adjust/adjust_helpers.py,sha256=-tmmxy9k3wms-ZEIgxmlp2cAQ2X_O1lgjY1128bbMu4,3224
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
@@ -27,6 +27,13 @@ ingestr/src/facebook_ads/settings.py,sha256=1IxZeP_4rN3IBvAncNHOoqpzAirx0Hz-MUK_
 ingestr/src/filesystem/__init__.py,sha256=wHHaKFuAjsR_ZRjl6g_Flf6FhVs9qhwREthTr03_7cc,4162
 ingestr/src/filesystem/helpers.py,sha256=bg0muSHZr3hMa8H4jN2-LGWzI-SUoKlQNiWJ74-YYms,3211
 ingestr/src/filesystem/readers.py,sha256=a0fKkaRpnAOGsXI3EBNYZa7x6tlmAOsgRzb883StY30,3987
+ingestr/src/github/__init__.py,sha256=csA2VcjOxXrVrvp7zY-JodO9Lpy98bJ4AqRdHCLTcGM,5838
+ingestr/src/github/helpers.py,sha256=Tmnik9811zBWNO6cJwV9PFQxEx2j32LHAQCvNbubsEI,6759
+ingestr/src/github/queries.py,sha256=W34C02jUEdjFmOE7f7u9xvYyBNDMfVZAu0JIRZI2mkU,2302
+ingestr/src/github/settings.py,sha256=N5ahWrDIQ_4IWV9i-hTXxyYduqY9Ym2BTwqsWxcDdJ8,258
+ingestr/src/google_analytics/__init__.py,sha256=HjA13wfJm2MGfy3h_DiM5ekkNqM2dgwYCKJ3pprnDtI,2482
+ingestr/src/google_analytics/helpers/__init__.py,sha256=y_q7dinlEwNBEpq6kCzjTa8lAhe2bb23bDPP0fcy7fY,2744
+ingestr/src/google_analytics/helpers/data_processing.py,sha256=fIdEKr9CmZN_s1T2i9BL8IYTPPqNoK6Vaquq2y8StfE,6072
 ingestr/src/google_sheets/README.md,sha256=wFQhvmGpRA38Ba2N_WIax6duyD4c7c_pwvvprRfQDnw,5470
 ingestr/src/google_sheets/__init__.py,sha256=5qlX-6ilx5MW7klC7B_0jGSxloQSLkSESTh4nlY3Aos,6643
 ingestr/src/google_sheets/helpers/__init__.py,sha256=5hXZrZK8cMO3UOuL-s4OKOpdACdihQD0hYYlSEu-iQ8,35
@@ -61,8 +68,8 @@ ingestr/src/stripe_analytics/helpers.py,sha256=iqZOyiGIOhOAhVXXU16DP0hkkTKcTrDu6
 ingestr/src/stripe_analytics/settings.py,sha256=rl9L5XumxO0pjkZf7MGesXHp4QLRgnz3RWLuDWDBKXo,380
 ingestr/src/telemetry/event.py,sha256=MpWc5tt0lSJ1pWKe9HQ11BHrcPBxSH40l4wjZi9u0tI,924
 ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
-ingestr/src/tiktok_ads/__init__.py,sha256=vJjVxEw3W1Rvc2QDQbox_8Ma0Cp1RT7iKsQ9MAv6Cgc,3036
-ingestr/src/tiktok_ads/tiktok_helpers.py,sha256=lY7yWl_aJh5Hj-bVvt07MHvhfvXnghaGOLhGHF5gLh4,3444
+ingestr/src/tiktok_ads/__init__.py,sha256=U4ZHPUW0c4LpKx4hjT2Lz5hgWFgwQSbAAkkYIrxYHZo,4469
+ingestr/src/tiktok_ads/tiktok_helpers.py,sha256=cfdPflCeR_mCk5fxq0v4d7pzlvZDiAoz8bWQJYqKALM,3935
 ingestr/src/zendesk/__init__.py,sha256=C7HkN195DGdOHId2_Sa_kAlcBrUmnVYZUa_tPkiyf1Q,17564
 ingestr/src/zendesk/settings.py,sha256=Vdj706nTJFQ-3KH4nO97iYCQuba3dV3E9gfnmLK6xwU,2294
 ingestr/src/zendesk/helpers/__init__.py,sha256=YTJejCiUjfIcsj9FrkY0l-JGYDI7RRte1Ydq5FDH_0c,888
@@ -77,8 +84,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.12.2.dist-info/METADATA,sha256=SAZJKqigL1ARQdv3eGX4RZVigZwYJCEcCt36lpvZtsQ,7910
-ingestr-0.12.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.12.2.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.12.2.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.12.2.dist-info/RECORD,,
+ingestr-0.12.4.dist-info/METADATA,sha256=VN9cqnH_rmALlSxePi6XOxOxndDGLYWTW0K6eafYVDw,7956
+ingestr-0.12.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.12.4.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.12.4.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.12.4.dist-info/RECORD,,