ingestr 0.13.54__py3-none-any.whl → 0.13.55__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.54"
+version = "v0.13.55"

ingestr/src/facebook_ads/__init__.py

@@ -116,6 +116,8 @@ def facebook_insights_source(
     batch_size: int = 50,
     request_timeout: int = 300,
     app_api_version: str = None,
+    start_date: pendulum.DateTime | None = None,
+    end_date: pendulum.DateTime | None = None,
 ) -> DltResource:
     """Incrementally loads insight reports with defined granularity level, fields, breakdowns etc.
 
@@ -148,27 +150,32 @@ def facebook_insights_source(
         account_id, access_token, request_timeout, app_api_version
     )
 
-    # we load with a defined lag
-    initial_load_start_date = pendulum.today().subtract(days=initial_load_past_days)
-    initial_load_start_date_str = initial_load_start_date.isoformat()
+    if start_date is None:
+        start_date = pendulum.today().subtract(days=initial_load_past_days)
+
+    columns = {}
+    for field in fields:
+        if field in INSIGHT_FIELDS_TYPES:
+            columns[field] = INSIGHT_FIELDS_TYPES[field]
 
     @dlt.resource(
         primary_key=INSIGHTS_PRIMARY_KEY,
         write_disposition="merge",
-        columns=INSIGHT_FIELDS_TYPES,
+        columns=columns,
     )
     def facebook_insights(
         date_start: dlt.sources.incremental[str] = dlt.sources.incremental(
             "date_start",
-            initial_value=initial_load_start_date_str,
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
             range_end="closed",
             range_start="closed",
+            lag=attribution_window_days_lag * 24 * 60 * 60,  # Convert days to seconds
         ),
     ) -> Iterator[TDataItems]:
-        start_date = get_start_date(date_start, attribution_window_days_lag)
+        start_date = get_start_date(date_start)
         end_date = pendulum.now()
 
-        # fetch insights in incremental day steps
         while start_date <= end_date:
             query = {
                 "level": level,
@@ -193,7 +200,10 @@ def facebook_insights_source(
                     }
                 ],
             }
-            job = execute_job(account.get_insights(params=query, is_async=True))
+            job = execute_job(
+                account.get_insights(params=query, is_async=True),
+                insights_max_async_sleep_seconds=10,
+            )
             yield list(map(process_report_item, job.get_result()))
             start_date = start_date.add(days=time_increment_days)
 

ingestr/src/facebook_ads/helpers.py

@@ -31,14 +31,13 @@ from .settings import (
 
 def get_start_date(
     incremental_start_date: dlt.sources.incremental[str],
-    attribution_window_days_lag: int = 7,
 ) -> pendulum.DateTime:
     """
     Get the start date for incremental loading of Facebook Insights data.
     """
     start_date: pendulum.DateTime = ensure_pendulum_datetime(
         incremental_start_date.start_value
-    ).subtract(days=attribution_window_days_lag)
+    )
 
     # facebook forgets insights so trim the lag and warn
     min_start_date = pendulum.today().subtract(
@@ -65,7 +64,6 @@ def process_report_item(item: AbstractObject) -> DictStrAny:
     for pki in INSIGHTS_PRIMARY_KEY:
         if pki not in d:
             d[pki] = "no_" + pki
-
     return d
 
 
@@ -138,7 +136,7 @@ def execute_job(
 ) -> AbstractCrudObject:
     status: str = None
     time_start = time.time()
-    sleep_time = 10
+    sleep_time = 3
     while status != "Job Completed":
         duration = time.time() - time_start
         job = job.api_get()

ingestr/src/facebook_ads/settings.py

@@ -112,6 +112,8 @@ DEFAULT_INSIGHT_FIELDS = (
     "social_spend",
     "spend",
     "website_ctr",
+    "conversions",
+    "video_thruplay_watched_actions",
 )
 
 TInsightsLevels = Literal["account", "campaign", "adset", "ad"]

ingestr/src/facebook_ads/utils.py

@@ -0,0 +1,39 @@
+from typing import Dict
+
+import dlt
+from dlt.common.configuration.inject import with_config
+from dlt.sources.helpers import requests
+
+
+@with_config(sections=("sources", "facebook_ads"))
+def debug_access_token(
+    access_token: str = dlt.secrets.value,
+    client_id: str = dlt.secrets.value,
+    client_secret: str = dlt.secrets.value,
+) -> str:
+    """Debugs the `access_token` providing info on expiration time, scopes etc. If arguments are not provides, `dlt` will inject them from configuration"""
+    debug_url = f"https://graph.facebook.com/debug_token?input_token={access_token}&access_token={client_id}|{client_secret}"
+    response = requests.get(debug_url)
+    data: Dict[str, str] = response.json()
+
+    if "error" in data:
+        raise Exception(f"Error debugging token: {data['error']}")
+
+    return data["data"]
+
+
+@with_config(sections=("sources", "facebook_ads"))
+def get_long_lived_token(
+    access_token: str = dlt.secrets.value,
+    client_id: str = dlt.secrets.value,
+    client_secret: str = dlt.secrets.value,
+) -> str:
+    """Gets the long lived access token (60 days) from `access_token`. If arguments are not provides, `dlt` will inject them from configuration"""
+    exchange_url = f"https://graph.facebook.com/v13.0/oauth/access_token?grant_type=fb_exchange_token&client_id={client_id}&client_secret={client_secret}&fb_exchange_token={access_token}"
+    response = requests.get(exchange_url)
+    data: Dict[str, str] = response.json()
+
+    if "error" in data:
+        raise Exception(f"Error refreshing token: {data['error']}")
+
+    return data["access_token"]

ingestr/src/sources.py

@@ -747,11 +747,64 @@ class FacebookAdsSource:
         endpoint = None
         if table in ["campaigns", "ad_sets", "ad_creatives", "ads", "leads"]:
             endpoint = table
-        elif table in "facebook_insights":
+        elif table == "facebook_insights":
             return facebook_insights_source(
                 access_token=access_token[0],
                 account_id=account_id[0],
+                start_date=kwargs.get("interval_start"),
+                end_date=kwargs.get("interval_end"),
             ).with_resources("facebook_insights")
+        elif table.startswith("facebook_insights:"):
+            # Parse custom breakdowns and metrics from table name
+            # Supported formats:
+            # facebook_insights:breakdown_type
+            # facebook_insights:breakdown_type:metric1,metric2...
+            parts = table.split(":")
+
+            if len(parts) < 2 or len(parts) > 3:
+                raise ValueError(
+                    "Invalid facebook_insights format. Expected: facebook_insights:breakdown_type or facebook_insights:breakdown_type:metric1,metric2..."
+                )
+
+            breakdown_type = parts[1].strip()
+            if not breakdown_type:
+                raise ValueError(
+                    "Breakdown type must be provided in format: facebook_insights:breakdown_type"
+                )
+
+            # Validate breakdown type against available options from settings
+            import typing
+
+            from ingestr.src.facebook_ads.settings import TInsightsBreakdownOptions
+
+            # Get valid breakdown options from the type definition
+            valid_breakdowns = list(typing.get_args(TInsightsBreakdownOptions))
+
+            if breakdown_type not in valid_breakdowns:
+                raise ValueError(
+                    f"Invalid breakdown type '{breakdown_type}'. Valid options: {', '.join(valid_breakdowns)}"
+                )
+
+            source_kwargs = {
+                "access_token": access_token[0],
+                "account_id": account_id[0],
+                "start_date": kwargs.get("interval_start"),
+                "end_date": kwargs.get("interval_end"),
+                "breakdowns": breakdown_type,
+            }
+
+            # If custom metrics are provided, parse them
+            if len(parts) == 3:
+                fields = [f.strip() for f in parts[2].split(",") if f.strip()]
+                if not fields:
+                    raise ValueError(
+                        "Custom metrics must be provided after the second colon in format: facebook_insights:breakdown_type:metric1,metric2..."
+                    )
+                source_kwargs["fields"] = fields
+
+            return facebook_insights_source(**source_kwargs).with_resources(
+                "facebook_insights"
+            )
         else:
             raise ValueError(
                 f"Resource '{table}' is not supported for Facebook Ads source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"

ingestr/src/stripe_analytics/__init__.py

@@ -85,12 +85,14 @@ def incremental_stripe_source(
         created: Optional[Any] = dlt.sources.incremental(
             "created",
             initial_value=start_date_unix,
+            end_value=transform_date(end_date) if end_date is not None else None,
             range_end="closed",
             range_start="closed",
         ),
     ) -> Generator[Dict[Any, Any], Any, None]:
-        start_value = created.last_value
-        yield from pagination(endpoint, start_date=start_value, end_date=end_date)
+        yield from pagination(
+            endpoint, start_date=created.last_value, end_date=created.end_value
+        )
 
     for endpoint in endpoints:
         yield dlt.resource(

{ingestr-0.13.54.dist-info → ingestr-0.13.55.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.54
+Version: 0.13.55
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.54.dist-info → ingestr-0.13.55.dist-info}/RECORD

@@ -2,7 +2,7 @@ ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
 ingestr/main.py,sha256=GkC1hdq8AVGrvolc95zMfjmibI95p2pmFkbgCOVf-Og,26311
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=5aqjzKmdujvelIHbUBjqDKLXcEB2hEdLvQ6BJyTyZ1Q,21
+ingestr/src/buildinfo.py,sha256=bdi0-mZhnHheYgs6WuEb8p-RIk_RFAXRCF9HalRfV0k,21
 ingestr/src/destinations.py,sha256=TcxM2rcwHfgMMP6U0yRNcfWKlEzkBbZbqCIDww7lkTY,16882
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
 ingestr/src/factory.py,sha256=mcjgbmrZr6TvP9fCMQxo-aMGcrb2PqToRcSLp5nldww,6138
@@ -11,7 +11,7 @@ ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,73
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=Nwx66jQwqjovLomZIVwIVNLSBy8KgqGhu4-spPHp7ZM,91785
+ingestr/src/sources.py,sha256=3ozLt9lhhNANspfjA2vb8u6qjgBJezH8QBV1XKqT1fg,94124
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -39,10 +39,11 @@ ingestr/src/chess/settings.py,sha256=p0RlCGgtXUacPDEvZmwzSWmzX0Apj1riwfz-nrMK89k
 ingestr/src/collector/spinner.py,sha256=_ZUqF5MI43hVIULdjF5s5mrAZbhEFXaiWirQmrv3Yk4,1201
 ingestr/src/dynamodb/__init__.py,sha256=swhxkeYBbJ35jn1IghCtvYWT2BM33KynVCh_oR4z28A,2264
 ingestr/src/elasticsearch/__init__.py,sha256=m-q93HgUmTwGDUwHOjHawstWL06TC3WIX3H05szybrY,2556
-ingestr/src/facebook_ads/__init__.py,sha256=reEpSr4BaKA1wO3qVgCH51gW-TgWkbJ_g24UIhJWbac,9286
+ingestr/src/facebook_ads/__init__.py,sha256=a1A5fO1r_FotoH9UET42tamqo_-ftCm9vBrkm5lpjG0,9579
 ingestr/src/facebook_ads/exceptions.py,sha256=4Nlbc0Mv3i5g-9AoyT-n1PIa8IDi3VCTfEAzholx4Wc,115
-ingestr/src/facebook_ads/helpers.py,sha256=ZLbNHiKer5lPb4g3_435XeBJr57Wv0o1KTyBA1mQ100,9068
-ingestr/src/facebook_ads/settings.py,sha256=1IxZeP_4rN3IBvAncNHOoqpzAirx0Hz-MUK_tl6UTFk,4881
+ingestr/src/facebook_ads/helpers.py,sha256=EYqOAPUlhVNxwzjP_CUGjJvAXTq65nJC-v75BfyJKmg,8981
+ingestr/src/facebook_ads/settings.py,sha256=Bsic8RcmH-NfEZ7r_NGospTCmwISK9XaMT5y2NZirtg,4938
+ingestr/src/facebook_ads/utils.py,sha256=ES2ylPoW3j3fjp6OMUgp21n1cG1OktXsmWWMk5vBW_I,1590
 ingestr/src/filesystem/__init__.py,sha256=zkIwbRr0ir0EUdniI25p2zGiVc-7M9EmR351AjNb0eA,4163
 ingestr/src/filesystem/helpers.py,sha256=bg0muSHZr3hMa8H4jN2-LGWzI-SUoKlQNiWJ74-YYms,3211
 ingestr/src/filesystem/readers.py,sha256=a0fKkaRpnAOGsXI3EBNYZa7x6tlmAOsgRzb883StY30,3987
@@ -116,7 +117,7 @@ ingestr/src/solidgate/__init__.py,sha256=JdaXvAu5QGuf9-FY294vwCQCEmfrqIld9oqbzqC
 ingestr/src/solidgate/helpers.py,sha256=oePEc9nnvmN3IaKrfJCvyKL79xdGM0-gRTN3-8tY4Fc,4952
 ingestr/src/sql_database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestr/src/sql_database/callbacks.py,sha256=sEFFmXxAURY3yeBjnawigDtq9LBCvi8HFqG4kLd7tMU,2002
-ingestr/src/stripe_analytics/__init__.py,sha256=FBkZu5op5Z-FceEi4zG7qcAgZfUYJRPMVPPrPMjvmXw,4502
+ingestr/src/stripe_analytics/__init__.py,sha256=j3Vmvo8G75fJJIF4rUnpGliGTpYQZt372wo-AjGImYs,4581
 ingestr/src/stripe_analytics/helpers.py,sha256=iqZOyiGIOhOAhVXXU16DP0hkkTKcTrDu69vAJoTxgEo,1976
 ingestr/src/stripe_analytics/settings.py,sha256=ZahhZg3Sq2KnvnDcfSaXO494Csy3tElBDEHnvA1AVmA,2461
 ingestr/src/telemetry/event.py,sha256=W7bs4uVfPakQ5otmiqgqu1l5SqjYx1p87wudnWXckBc,949
@@ -138,8 +139,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
-ingestr-0.13.54.dist-info/METADATA,sha256=5RaRSmJX-SiZV9iFYz3cSkiuZp0GXQ2ij9whkbQHFs8,15131
-ingestr-0.13.54.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.54.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.54.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.54.dist-info/RECORD,,
+ingestr-0.13.55.dist-info/METADATA,sha256=WNMM4qLCTDJg4xUnYNefHffB6vidRl4xopoBaaux-FM,15131
+ingestr-0.13.55.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.55.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.55.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.55.dist-info/RECORD,,