ingestr 0.13.80__tar.gz → 0.13.81__tar.gz

{ingestr-0.13.80 → ingestr-0.13.81}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.80
+Version: 0.13.81
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.80 → ingestr-0.13.81}/docs/supported-sources/facebook-ads.md

@@ -160,13 +160,19 @@ The `facebook_insights` table supports advanced configuration for breakdowns and
 1. **Default usage**: `facebook_insights`
    - Uses default breakdown and default fields
 
-2. **Custom breakdown**: `facebook_insights:breakdown_type`
-   - Uses specified breakdown with default fields
+2. **Predefined breakdown**: `facebook_insights:breakdown_type`
+   - Uses specified predefined breakdown with default fields
 
-3. **Custom breakdown + metrics**: `facebook_insights:breakdown_type:metric1,metric2,metric3`
-   - Uses specified breakdown with custom metrics
+3. **Predefined breakdown + custom metrics**: `facebook_insights:breakdown_type:metric1,metric2,metric3`
+   - Uses specified predefined breakdown with custom metrics
 
-#### Available Breakdown Types
+4. **Custom dimensions + metrics**: `facebook_insights:dimension1,dimension2:metric1,metric2,metric3`
+   - Uses custom dimensions with custom metrics
+
+5. **Level + dimensions + metrics**: `facebook_insights:level,dimension1,dimension2:metric1,metric2,metric3`
+   - Uses specified level with custom dimensions and metrics
+
+#### Available Predefined Breakdown Types
 
 - `ads_insights` (default)
 - `ads_insights_age_and_gender`
@@ -176,6 +182,30 @@ The `facebook_insights` table supports advanced configuration for breakdowns and
 - `ads_insights_dma`
 - `ads_insights_hourly_advertiser`
 
+#### Available Levels
+
+When using custom dimensions, you can specify one of these levels:
+- `account` - Account level insights
+- `campaign` - Campaign level insights  
+- `adset` - Ad set level insights
+- `ad` - Ad level insights
+
+Note: If multiple levels are specified in the dimensions list, the last valid level will be used and removed from the dimensions list.
+
+#### Common Dimensions
+
+You can use any valid Facebook Ads dimension in your custom configurations. Some commonly used dimensions include:
+- `age` - Age ranges
+- `gender` - Gender breakdown
+- `country` - Country breakdown
+- `region` - Region breakdown  
+- `platform_position` - Platform position
+- `publisher_platform` - Publisher platform
+- `impression_device` - Device type
+- `placement` - Ad placement
+
+Note: Not all dimension combinations are valid according to Facebook's API. Refer to [Facebook's Marketing API](https://developers.facebook.com/docs/marketing-api/insights/breakdowns/) documentation for valid dimension combinations.
+
 #### Examples
 
 ```sh
@@ -199,4 +229,32 @@ ingestr ingest \
   --source-table 'facebook_insights:ads_insights_country:impressions,clicks,spend,reach,cpm,ctr' \
   --dest-uri 'duckdb:///facebook.duckdb' \
   --dest-table 'dest.insights_by_country'
+
+# Custom dimensions (age and gender) with custom metrics
+ingestr ingest \
+  --source-uri 'facebookads://?access_token=easdyh&account_id=1234' \
+  --source-table 'facebook_insights:age,gender:impressions,clicks,spend' \
+  --dest-uri 'duckdb:///facebook.duckdb' \
+  --dest-table 'dest.insights_custom_dimensions'
+
+# Campaign level with custom dimensions and metrics
+ingestr ingest \
+  --source-uri 'facebookads://?access_token=easdyh&account_id=1234' \
+  --source-table 'facebook_insights:campaign,age,gender:impressions,clicks,spend,reach' \
+  --dest-uri 'duckdb:///facebook.duckdb' \
+  --dest-table 'dest.campaign_insights_demographics'
+
+# Ad level with geographic dimensions
+ingestr ingest \
+  --source-uri 'facebookads://?access_token=easdyh&account_id=1234' \
+  --source-table 'facebook_insights:ad,country,region:clicks,impressions,spend' \
+  --dest-uri 'duckdb:///facebook.duckdb' \
+  --dest-table 'dest.ad_insights_geographic'
+
+# Account level insights only (no additional dimensions)
+ingestr ingest \
+  --source-uri 'facebookads://?access_token=easdyh&account_id=1234' \
+  --source-table 'facebook_insights:account:impressions,clicks,spend,reach' \
+  --dest-uri 'duckdb:///facebook.duckdb' \
+  --dest-table 'dest.account_level_insights'
 ```

ingestr-0.13.81/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.81"

{ingestr-0.13.80 → ingestr-0.13.81}/ingestr/src/facebook_ads/__init__.py

@@ -22,12 +22,8 @@ from .settings import (
     DEFAULT_ADCREATIVE_FIELDS,
     DEFAULT_ADSET_FIELDS,
     DEFAULT_CAMPAIGN_FIELDS,
-    DEFAULT_INSIGHT_FIELDS,
     DEFAULT_LEAD_FIELDS,
     INSIGHT_FIELDS_TYPES,
-    INSIGHTS_BREAKDOWNS_OPTIONS,
-    INVALID_INSIGHTS_FIELDS,
-    TInsightsBreakdownOptions,
     TInsightsLevels,
 )
 
@@ -105,10 +101,9 @@ def facebook_insights_source(
     account_id: str = dlt.config.value,
     access_token: str = dlt.secrets.value,
     initial_load_past_days: int = 1,
-    fields: Sequence[str] = DEFAULT_INSIGHT_FIELDS,
-    attribution_window_days_lag: int = 7,
+    dimensions: Sequence[str] = None,
+    fields: Sequence[str] = None,
     time_increment_days: int = 1,
-    breakdowns: TInsightsBreakdownOptions = "ads_insights",
     action_breakdowns: Sequence[str] = ALL_ACTION_BREAKDOWNS,
     level: TInsightsLevels = "ad",
     action_attribution_windows: Sequence[str] = ALL_ACTION_ATTRIBUTION_WINDOWS,
@@ -155,6 +150,11 @@ def facebook_insights_source(
     if start_date is None:
         start_date = pendulum.today().subtract(days=initial_load_past_days)
 
+    if dimensions is None:
+        dimensions = []
+    if fields is None:
+        fields = []
+
     columns = {}
     for field in fields:
         if field in INSIGHT_FIELDS_TYPES:
@@ -187,15 +187,9 @@ def facebook_insights_source(
             query = {
                 "level": level,
                 "action_breakdowns": list(action_breakdowns),
-                "breakdowns": list(
-                    INSIGHTS_BREAKDOWNS_OPTIONS[breakdowns]["breakdowns"]
-                ),
+                "breakdowns": dimensions,
                 "limit": batch_size,
-                "fields": list(
-                    set(fields)
-                    .union(INSIGHTS_BREAKDOWNS_OPTIONS[breakdowns]["fields"])
-                    .difference(INVALID_INSIGHTS_FIELDS)
-                ),
+                "fields": fields,
                 "time_increment": time_increment_days,
                 "action_attribution_windows": list(action_attribution_windows),
                 "time_ranges": [

{ingestr-0.13.80 → ingestr-0.13.81}/ingestr/src/facebook_ads/helpers.py

@@ -229,3 +229,49 @@ def notify_on_token_expiration(access_token_expires_at: int = None) -> None:
             logger.error(
                 f"Access Token expires in {humanize.precisedelta(pendulum.now() - expires_at)}. Replace the token now!"
             )
+
+
+def parse_insights_table_to_source_kwargs(table: str) -> DictStrAny:
+    import typing
+
+    from ingestr.src.facebook_ads.settings import (
+        INSIGHTS_BREAKDOWNS_OPTIONS,
+        TInsightsBreakdownOptions,
+        TInsightsLevels,
+    )
+
+    parts = table.split(":")
+
+    source_kwargs = {}
+
+    breakdown_type = parts[1]
+
+    valid_breakdowns = list(typing.get_args(TInsightsBreakdownOptions))
+    if breakdown_type in valid_breakdowns:
+        dimensions = INSIGHTS_BREAKDOWNS_OPTIONS[breakdown_type]["breakdowns"]
+        fields = INSIGHTS_BREAKDOWNS_OPTIONS[breakdown_type]["fields"]
+        source_kwargs["dimensions"] = dimensions
+        source_kwargs["fields"] = fields
+    else:
+        dimensions = breakdown_type.split(",")
+        valid_levels = list(typing.get_args(TInsightsLevels))
+        level = None
+        for valid_level in reversed(valid_levels):
+            if valid_level in dimensions:
+                level = valid_level
+                dimensions.remove(valid_level)
+                break
+
+        source_kwargs["level"] = level
+        source_kwargs["dimensions"] = dimensions
+
+    # If custom metrics are provided, parse them
+    if len(parts) == 3:
+        fields = [f.strip() for f in parts[2].split(",") if f.strip()]
+        if not fields:
+            raise ValueError(
+                "Custom metrics must be provided after the second colon in format: facebook_insights:breakdown_type:metric1,metric2..."
+            )
+        source_kwargs["fields"] = fields
+
+    return source_kwargs

{ingestr-0.13.80 → ingestr-0.13.81}/ingestr/src/linear/__init__.py

@@ -17,9 +17,10 @@ def _get_date_range(updated_at, start_date):
         current_end_date = pendulum.parse(updated_at.end_value)
     else:
         current_end_date = pendulum.now(tz="UTC")
-    
+
     return current_start_date, current_end_date
 
+
 ISSUES_QUERY = """
 query Issues($cursor: String) {
   issues(first: 50, after: $cursor) {
@@ -119,6 +120,7 @@ query WorkflowStates($cursor: String) {
 }
 """
 
+
 @dlt.source(name="linear", max_table_nesting=0)
 def linear_source(
     api_key: str,
@@ -211,5 +213,5 @@ def linear_source(
             if pendulum.parse(item["updatedAt"]) >= current_start_date:
                 if pendulum.parse(item["updatedAt"]) <= current_end_date:
                     yield normalize_dictionaries(item)
-    return [issues, projects, teams, users, workflow_states]
 
+    return [issues, projects, teams, users, workflow_states]

{ingestr-0.13.80 → ingestr-0.13.81}/ingestr/src/linear/helpers.py

@@ -32,8 +32,6 @@ def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
         cursor = data["pageInfo"]["endCursor"]
 
 
-
-
 def normalize_dictionaries(item: Dict[str, Any]) -> Dict[str, Any]:
     """
     Automatically normalize dictionary fields by detecting their structure:
@@ -41,15 +39,15 @@ def normalize_dictionaries(item: Dict[str, Any]) -> Dict[str, Any]:
     - Convert objects with 'nodes' field to arrays
     """
     normalized_item = item.copy()
-    
+
     for key, value in list(normalized_item.items()):
         if isinstance(value, dict):
             # If the dict has an 'id' field, replace with {key}_id
-            if 'id' in value:
-                normalized_item[f"{key}_id"] = value['id']
+            if "id" in value:
+                normalized_item[f"{key}_id"] = value["id"]
                 del normalized_item[key]
             # If the dict has 'nodes' field, extract the nodes array
-            elif 'nodes' in value:
-                normalized_item[key] = value['nodes']
-    
+            elif "nodes" in value:
+                normalized_item[key] = value["nodes"]
+
     return normalized_item

{ingestr-0.13.80 → ingestr-0.13.81}/ingestr/src/sources.py

@@ -1056,33 +1056,16 @@ class FacebookAdsSource:
             # Validate breakdown type against available options from settings
             import typing
 
-            from ingestr.src.facebook_ads.settings import TInsightsBreakdownOptions
-
-            # Get valid breakdown options from the type definition
-            valid_breakdowns = list(typing.get_args(TInsightsBreakdownOptions))
-
-            if breakdown_type not in valid_breakdowns:
-                raise ValueError(
-                    f"Invalid breakdown type '{breakdown_type}'. Valid options: {', '.join(valid_breakdowns)}"
-                )
+            from ingestr.src.facebook_ads.helpers import parse_insights_table_to_source_kwargs
 
             source_kwargs = {
                 "access_token": access_token[0],
                 "account_id": account_id[0],
                 "start_date": kwargs.get("interval_start"),
                 "end_date": kwargs.get("interval_end"),
-                "breakdowns": breakdown_type,
             }
 
-            # If custom metrics are provided, parse them
-            if len(parts) == 3:
-                fields = [f.strip() for f in parts[2].split(",") if f.strip()]
-                if not fields:
-                    raise ValueError(
-                        "Custom metrics must be provided after the second colon in format: facebook_insights:breakdown_type:metric1,metric2..."
-                    )
-                source_kwargs["fields"] = fields
-
+            source_kwargs.update(parse_insights_table_to_source_kwargs(table))
             return facebook_insights_source(**source_kwargs).with_resources(
                 "facebook_insights"
             )

ingestr-0.13.80/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.80"