ingestr 0.13.80__py3-none-any.whl → 0.13.81__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.80"
+version = "v0.13.81"

ingestr/src/facebook_ads/__init__.py

@@ -22,12 +22,8 @@ from .settings import (
     DEFAULT_ADCREATIVE_FIELDS,
     DEFAULT_ADSET_FIELDS,
     DEFAULT_CAMPAIGN_FIELDS,
-    DEFAULT_INSIGHT_FIELDS,
     DEFAULT_LEAD_FIELDS,
     INSIGHT_FIELDS_TYPES,
-    INSIGHTS_BREAKDOWNS_OPTIONS,
-    INVALID_INSIGHTS_FIELDS,
-    TInsightsBreakdownOptions,
     TInsightsLevels,
 )
 
@@ -105,10 +101,9 @@ def facebook_insights_source(
     account_id: str = dlt.config.value,
     access_token: str = dlt.secrets.value,
     initial_load_past_days: int = 1,
-    fields: Sequence[str] = DEFAULT_INSIGHT_FIELDS,
-    attribution_window_days_lag: int = 7,
+    dimensions: Sequence[str] = None,
+    fields: Sequence[str] = None,
     time_increment_days: int = 1,
-    breakdowns: TInsightsBreakdownOptions = "ads_insights",
     action_breakdowns: Sequence[str] = ALL_ACTION_BREAKDOWNS,
     level: TInsightsLevels = "ad",
     action_attribution_windows: Sequence[str] = ALL_ACTION_ATTRIBUTION_WINDOWS,
@@ -155,6 +150,11 @@ def facebook_insights_source(
     if start_date is None:
         start_date = pendulum.today().subtract(days=initial_load_past_days)
 
+    if dimensions is None:
+        dimensions = []
+    if fields is None:
+        fields = []
+
     columns = {}
     for field in fields:
         if field in INSIGHT_FIELDS_TYPES:
@@ -187,15 +187,9 @@ def facebook_insights_source(
             query = {
                 "level": level,
                 "action_breakdowns": list(action_breakdowns),
-                "breakdowns": list(
-                    INSIGHTS_BREAKDOWNS_OPTIONS[breakdowns]["breakdowns"]
-                ),
+                "breakdowns": dimensions,
                 "limit": batch_size,
-                "fields": list(
-                    set(fields)
-                    .union(INSIGHTS_BREAKDOWNS_OPTIONS[breakdowns]["fields"])
-                    .difference(INVALID_INSIGHTS_FIELDS)
-                ),
+                "fields": fields,
                 "time_increment": time_increment_days,
                 "action_attribution_windows": list(action_attribution_windows),
                 "time_ranges": [

ingestr/src/facebook_ads/helpers.py

@@ -229,3 +229,49 @@ def notify_on_token_expiration(access_token_expires_at: int = None) -> None:
             logger.error(
                 f"Access Token expires in {humanize.precisedelta(pendulum.now() - expires_at)}. Replace the token now!"
             )
+
+
+def parse_insights_table_to_source_kwargs(table: str) -> DictStrAny:
+    import typing
+
+    from ingestr.src.facebook_ads.settings import (
+        INSIGHTS_BREAKDOWNS_OPTIONS,
+        TInsightsBreakdownOptions,
+        TInsightsLevels,
+    )
+
+    parts = table.split(":")
+
+    source_kwargs = {}
+
+    breakdown_type = parts[1]
+
+    valid_breakdowns = list(typing.get_args(TInsightsBreakdownOptions))
+    if breakdown_type in valid_breakdowns:
+        dimensions = INSIGHTS_BREAKDOWNS_OPTIONS[breakdown_type]["breakdowns"]
+        fields = INSIGHTS_BREAKDOWNS_OPTIONS[breakdown_type]["fields"]
+        source_kwargs["dimensions"] = dimensions
+        source_kwargs["fields"] = fields
+    else:
+        dimensions = breakdown_type.split(",")
+        valid_levels = list(typing.get_args(TInsightsLevels))
+        level = None
+        for valid_level in reversed(valid_levels):
+            if valid_level in dimensions:
+                level = valid_level
+                dimensions.remove(valid_level)
+                break
+
+        source_kwargs["level"] = level
+        source_kwargs["dimensions"] = dimensions
+
+    # If custom metrics are provided, parse them
+    if len(parts) == 3:
+        fields = [f.strip() for f in parts[2].split(",") if f.strip()]
+        if not fields:
+            raise ValueError(
+                "Custom metrics must be provided after the second colon in format: facebook_insights:breakdown_type:metric1,metric2..."
+            )
+        source_kwargs["fields"] = fields
+
+    return source_kwargs

ingestr/src/linear/__init__.py

@@ -17,9 +17,10 @@ def _get_date_range(updated_at, start_date):
         current_end_date = pendulum.parse(updated_at.end_value)
     else:
         current_end_date = pendulum.now(tz="UTC")
-    
+
     return current_start_date, current_end_date
 
+
 ISSUES_QUERY = """
 query Issues($cursor: String) {
   issues(first: 50, after: $cursor) {
@@ -119,6 +120,7 @@ query WorkflowStates($cursor: String) {
 }
 """
 
+
 @dlt.source(name="linear", max_table_nesting=0)
 def linear_source(
     api_key: str,
@@ -211,5 +213,5 @@ def linear_source(
             if pendulum.parse(item["updatedAt"]) >= current_start_date:
                 if pendulum.parse(item["updatedAt"]) <= current_end_date:
                     yield normalize_dictionaries(item)
-    return [issues, projects, teams, users, workflow_states]
 
+    return [issues, projects, teams, users, workflow_states]

ingestr/src/linear/helpers.py

@@ -32,8 +32,6 @@ def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
         cursor = data["pageInfo"]["endCursor"]
 
 
-
-
 def normalize_dictionaries(item: Dict[str, Any]) -> Dict[str, Any]:
     """
     Automatically normalize dictionary fields by detecting their structure:
@@ -41,15 +39,15 @@ def normalize_dictionaries(item: Dict[str, Any]) -> Dict[str, Any]:
     - Convert objects with 'nodes' field to arrays
     """
     normalized_item = item.copy()
-    
+
     for key, value in list(normalized_item.items()):
         if isinstance(value, dict):
             # If the dict has an 'id' field, replace with {key}_id
-            if 'id' in value:
-                normalized_item[f"{key}_id"] = value['id']
+            if "id" in value:
+                normalized_item[f"{key}_id"] = value["id"]
                 del normalized_item[key]
             # If the dict has 'nodes' field, extract the nodes array
-            elif 'nodes' in value:
-                normalized_item[key] = value['nodes']
-    
+            elif "nodes" in value:
+                normalized_item[key] = value["nodes"]
+
     return normalized_item

ingestr/src/sources.py

@@ -1056,33 +1056,16 @@ class FacebookAdsSource:
             # Validate breakdown type against available options from settings
             import typing
 
-            from ingestr.src.facebook_ads.settings import TInsightsBreakdownOptions
-
-            # Get valid breakdown options from the type definition
-            valid_breakdowns = list(typing.get_args(TInsightsBreakdownOptions))
-
-            if breakdown_type not in valid_breakdowns:
-                raise ValueError(
-                    f"Invalid breakdown type '{breakdown_type}'. Valid options: {', '.join(valid_breakdowns)}"
-                )
+            from ingestr.src.facebook_ads.helpers import parse_insights_table_to_source_kwargs
 
             source_kwargs = {
                 "access_token": access_token[0],
                 "account_id": account_id[0],
                 "start_date": kwargs.get("interval_start"),
                 "end_date": kwargs.get("interval_end"),
-                "breakdowns": breakdown_type,
             }
 
-            # If custom metrics are provided, parse them
-            if len(parts) == 3:
-                fields = [f.strip() for f in parts[2].split(",") if f.strip()]
-                if not fields:
-                    raise ValueError(
-                        "Custom metrics must be provided after the second colon in format: facebook_insights:breakdown_type:metric1,metric2..."
-                    )
-                source_kwargs["fields"] = fields
-
+            source_kwargs.update(parse_insights_table_to_source_kwargs(table))
             return facebook_insights_source(**source_kwargs).with_resources(
                 "facebook_insights"
             )

{ingestr-0.13.80.dist-info → ingestr-0.13.81.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.80
+Version: 0.13.81
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.80.dist-info → ingestr-0.13.81.dist-info}/RECORD

@@ -2,7 +2,7 @@ ingestr/conftest.py,sha256=OE2yxeTCosS9CUFVuqNypm-2ftYvVBeeq7egm3878cI,1981
 ingestr/main.py,sha256=qoWHNcHh0-xVnyQxbQ-SKuTxPb1RNV3ENkCpqO7CLrk,26694
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
-ingestr/src/buildinfo.py,sha256=K8l-cnsoOmTtSy1GdxNMRLZxsjHoYiJsCc5rvXeCpHE,21
+ingestr/src/buildinfo.py,sha256=H6wsXWMjlzTHcGuLKaXl67mkAFCFJzin5gNxWfo6nMg,21
 ingestr/src/destinations.py,sha256=M2Yni6wiWcrvZ8EPJemidqxN156l0rehgCc7xuil7mo,22840
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
 ingestr/src/factory.py,sha256=rF5Ry4o4t8KulSPBtrd7ZKCI_0TH1DAetG0zs9H7oik,6792
@@ -11,7 +11,7 @@ ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,73
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=guwCdKQDvT2XMYeR2O3nJ9kZ-wLCSDpOex4UH0luG5k,119966
+ingestr/src/sources.py,sha256=wdNRqVzuR8EK8s8SzL3lZ8UE6UrqLMq-FJGkc23rrpM,119190
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -41,9 +41,9 @@ ingestr/src/clickup/helpers.py,sha256=RzDKMUAHccuDhocIQ2ToBXfCERo8CBJqA3t-IPltBC
 ingestr/src/collector/spinner.py,sha256=_ZUqF5MI43hVIULdjF5s5mrAZbhEFXaiWirQmrv3Yk4,1201
 ingestr/src/dynamodb/__init__.py,sha256=swhxkeYBbJ35jn1IghCtvYWT2BM33KynVCh_oR4z28A,2264
 ingestr/src/elasticsearch/__init__.py,sha256=m-q93HgUmTwGDUwHOjHawstWL06TC3WIX3H05szybrY,2556
-ingestr/src/facebook_ads/__init__.py,sha256=bX6lnf0LxIcOyZHDVA9FL5iKhgnQ0f5Hfma4eXcQuIk,10094
+ingestr/src/facebook_ads/__init__.py,sha256=15GiovITANe0al5MI6WWLdl3LDmdBd1YpkUWBV3g6bk,9715
 ingestr/src/facebook_ads/exceptions.py,sha256=4Nlbc0Mv3i5g-9AoyT-n1PIa8IDi3VCTfEAzholx4Wc,115
-ingestr/src/facebook_ads/helpers.py,sha256=Oh9-LepxxBRnPXQZMbbNOSbxg9T8a4nmiLSt22GPt6E,8233
+ingestr/src/facebook_ads/helpers.py,sha256=c-WG008yU_zIdhFwljtqE2jfjVYuaVoNKldxcnJN3U4,9761
 ingestr/src/facebook_ads/settings.py,sha256=Bsic8RcmH-NfEZ7r_NGospTCmwISK9XaMT5y2NZirtg,4938
 ingestr/src/facebook_ads/utils.py,sha256=ES2ylPoW3j3fjp6OMUgp21n1cG1OktXsmWWMk5vBW_I,1590
 ingestr/src/filesystem/__init__.py,sha256=zkIwbRr0ir0EUdniI25p2zGiVc-7M9EmR351AjNb0eA,4163
@@ -85,8 +85,8 @@ ingestr/src/kinesis/helpers.py,sha256=SO2cFmWNGcykUYmjHdfxWsOQSkLQXyhFtfWnkcUOM0
 ingestr/src/klaviyo/__init__.py,sha256=o_noUgbxLk36s4f9W56_ibPorF0n7kVapPUlV0p-jfA,7875
 ingestr/src/klaviyo/client.py,sha256=tPj79ia7AW0ZOJhzlKNPCliGbdojRNwUFp8HvB2ym5s,7434
 ingestr/src/klaviyo/helpers.py,sha256=_i-SHffhv25feLDcjy6Blj1UxYLISCwVCMgGtrlnYHk,496
-ingestr/src/linear/__init__.py,sha256=XWgWiDJi87OFHnsOjTq4ZSAdtMcPdplBYC8fJM-6dMA,6607
-ingestr/src/linear/helpers.py,sha256=y8XhEDBVnxMmSzzyrS0_RnPwtNJIRuKM4Kw3wW9p6UM,1796
+ingestr/src/linear/__init__.py,sha256=CZnRQfP44L1fy4ppnukuTr1wWKSNlPpaLFE3Xw5vJr0,6605
+ingestr/src/linear/helpers.py,sha256=jpG9l3JhJVXk2K77iny8BMqkYHGycwS4Kmcdq2Xs7aE,1786
 ingestr/src/linkedin_ads/__init__.py,sha256=CAPWFyV24loziiphbLmODxZUXZJwm4JxlFkr56q0jfo,1855
 ingestr/src/linkedin_ads/dimension_time_enum.py,sha256=EmHRdkFyTAfo4chGjThrwqffWJxmAadZMbpTvf0xkQc,198
 ingestr/src/linkedin_ads/helpers.py,sha256=eUWudRVlXl4kqIhfXQ1eVsUpZwJn7UFqKSpnbLfxzds,4498
@@ -151,8 +151,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
-ingestr-0.13.80.dist-info/METADATA,sha256=HF4xO0hXDV_gn389MIuX_-GhahKPaz8GOMIMt1If4x0,15182
-ingestr-0.13.80.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.80.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.80.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.80.dist-info/RECORD,,
+ingestr-0.13.81.dist-info/METADATA,sha256=jjolsfHDoVMclLreJMhKFu6s1_FZ5d_eN2F8CFgEDFA,15182
+ingestr-0.13.81.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.81.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.81.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.81.dist-info/RECORD,,