ingestr 0.13.79__tar.gz → 0.13.81__tar.gz

{ingestr-0.13.79 → ingestr-0.13.81}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.79
+Version: 0.13.81
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.79 → ingestr-0.13.81}/docs/supported-sources/facebook-ads.md

@@ -160,13 +160,19 @@ The `facebook_insights` table supports advanced configuration for breakdowns and
 1. **Default usage**: `facebook_insights`
    - Uses default breakdown and default fields
 
-2. **Custom breakdown**: `facebook_insights:breakdown_type`
-   - Uses specified breakdown with default fields
+2. **Predefined breakdown**: `facebook_insights:breakdown_type`
+   - Uses specified predefined breakdown with default fields
 
-3. **Custom breakdown + metrics**: `facebook_insights:breakdown_type:metric1,metric2,metric3`
-   - Uses specified breakdown with custom metrics
+3. **Predefined breakdown + custom metrics**: `facebook_insights:breakdown_type:metric1,metric2,metric3`
+   - Uses specified predefined breakdown with custom metrics
 
-#### Available Breakdown Types
+4. **Custom dimensions + metrics**: `facebook_insights:dimension1,dimension2:metric1,metric2,metric3`
+   - Uses custom dimensions with custom metrics
+
+5. **Level + dimensions + metrics**: `facebook_insights:level,dimension1,dimension2:metric1,metric2,metric3`
+   - Uses specified level with custom dimensions and metrics
+
+#### Available Predefined Breakdown Types
 
 - `ads_insights` (default)
 - `ads_insights_age_and_gender`
@@ -176,6 +182,30 @@ The `facebook_insights` table supports advanced configuration for breakdowns and
 - `ads_insights_dma`
 - `ads_insights_hourly_advertiser`
 
+#### Available Levels
+
+When using custom dimensions, you can specify one of these levels:
+- `account` - Account level insights
+- `campaign` - Campaign level insights  
+- `adset` - Ad set level insights
+- `ad` - Ad level insights
+
+Note: If multiple levels are specified in the dimensions list, the last valid level will be used and removed from the dimensions list.
+
+#### Common Dimensions
+
+You can use any valid Facebook Ads dimension in your custom configurations. Some commonly used dimensions include:
+- `age` - Age ranges
+- `gender` - Gender breakdown
+- `country` - Country breakdown
+- `region` - Region breakdown  
+- `platform_position` - Platform position
+- `publisher_platform` - Publisher platform
+- `impression_device` - Device type
+- `placement` - Ad placement
+
+Note: Not all dimension combinations are valid according to Facebook's API. Refer to [Facebook's Marketing API](https://developers.facebook.com/docs/marketing-api/insights/breakdowns/) documentation for valid dimension combinations.
+
 #### Examples
 
 ```sh
@@ -199,4 +229,32 @@ ingestr ingest \
   --source-table 'facebook_insights:ads_insights_country:impressions,clicks,spend,reach,cpm,ctr' \
   --dest-uri 'duckdb:///facebook.duckdb' \
   --dest-table 'dest.insights_by_country'
+
+# Custom dimensions (age and gender) with custom metrics
+ingestr ingest \
+  --source-uri 'facebookads://?access_token=easdyh&account_id=1234' \
+  --source-table 'facebook_insights:age,gender:impressions,clicks,spend' \
+  --dest-uri 'duckdb:///facebook.duckdb' \
+  --dest-table 'dest.insights_custom_dimensions'
+
+# Campaign level with custom dimensions and metrics
+ingestr ingest \
+  --source-uri 'facebookads://?access_token=easdyh&account_id=1234' \
+  --source-table 'facebook_insights:campaign,age,gender:impressions,clicks,spend,reach' \
+  --dest-uri 'duckdb:///facebook.duckdb' \
+  --dest-table 'dest.campaign_insights_demographics'
+
+# Ad level with geographic dimensions
+ingestr ingest \
+  --source-uri 'facebookads://?access_token=easdyh&account_id=1234' \
+  --source-table 'facebook_insights:ad,country,region:clicks,impressions,spend' \
+  --dest-uri 'duckdb:///facebook.duckdb' \
+  --dest-table 'dest.ad_insights_geographic'
+
+# Account level insights only (no additional dimensions)
+ingestr ingest \
+  --source-uri 'facebookads://?access_token=easdyh&account_id=1234' \
+  --source-table 'facebook_insights:account:impressions,clicks,spend,reach' \
+  --dest-uri 'duckdb:///facebook.duckdb' \
+  --dest-table 'dest.account_level_insights'
 ```

ingestr-0.13.81/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.81"

{ingestr-0.13.79 → ingestr-0.13.81}/ingestr/src/facebook_ads/__init__.py

@@ -22,12 +22,8 @@ from .settings import (
     DEFAULT_ADCREATIVE_FIELDS,
     DEFAULT_ADSET_FIELDS,
     DEFAULT_CAMPAIGN_FIELDS,
-    DEFAULT_INSIGHT_FIELDS,
     DEFAULT_LEAD_FIELDS,
     INSIGHT_FIELDS_TYPES,
-    INSIGHTS_BREAKDOWNS_OPTIONS,
-    INVALID_INSIGHTS_FIELDS,
-    TInsightsBreakdownOptions,
     TInsightsLevels,
 )
 
@@ -105,10 +101,9 @@ def facebook_insights_source(
     account_id: str = dlt.config.value,
     access_token: str = dlt.secrets.value,
     initial_load_past_days: int = 1,
-    fields: Sequence[str] = DEFAULT_INSIGHT_FIELDS,
-    attribution_window_days_lag: int = 7,
+    dimensions: Sequence[str] = None,
+    fields: Sequence[str] = None,
     time_increment_days: int = 1,
-    breakdowns: TInsightsBreakdownOptions = "ads_insights",
     action_breakdowns: Sequence[str] = ALL_ACTION_BREAKDOWNS,
     level: TInsightsLevels = "ad",
     action_attribution_windows: Sequence[str] = ALL_ACTION_ATTRIBUTION_WINDOWS,
@@ -117,6 +112,9 @@ def facebook_insights_source(
     app_api_version: str = None,
     start_date: pendulum.DateTime | None = None,
     end_date: pendulum.DateTime | None = None,
+    insights_max_wait_to_finish_seconds: int = 60 * 60 * 4,
+    insights_max_wait_to_start_seconds: int = 60 * 30,
+    insights_max_async_sleep_seconds: int = 20,
 ) -> DltResource:
     """Incrementally loads insight reports with defined granularity level, fields, breakdowns etc.
 
@@ -152,6 +150,11 @@ def facebook_insights_source(
     if start_date is None:
         start_date = pendulum.today().subtract(days=initial_load_past_days)
 
+    if dimensions is None:
+        dimensions = []
+    if fields is None:
+        fields = []
+
     columns = {}
     for field in fields:
         if field in INSIGHT_FIELDS_TYPES:
@@ -184,15 +187,9 @@ def facebook_insights_source(
             query = {
                 "level": level,
                 "action_breakdowns": list(action_breakdowns),
-                "breakdowns": list(
-                    INSIGHTS_BREAKDOWNS_OPTIONS[breakdowns]["breakdowns"]
-                ),
+                "breakdowns": dimensions,
                 "limit": batch_size,
-                "fields": list(
-                    set(fields)
-                    .union(INSIGHTS_BREAKDOWNS_OPTIONS[breakdowns]["fields"])
-                    .difference(INVALID_INSIGHTS_FIELDS)
-                ),
+                "fields": fields,
                 "time_increment": time_increment_days,
                 "action_attribution_windows": list(action_attribution_windows),
                 "time_ranges": [
@@ -206,7 +203,9 @@ def facebook_insights_source(
             }
             job = execute_job(
                 account.get_insights(params=query, is_async=True),
-                insights_max_async_sleep_seconds=20,
+                insights_max_async_sleep_seconds=insights_max_async_sleep_seconds,
+                insights_max_wait_to_finish_seconds=insights_max_wait_to_finish_seconds,
+                insights_max_wait_to_start_seconds=insights_max_wait_to_start_seconds,
             )
             output = list(map(process_report_item, job.get_result()))
             yield output

{ingestr-0.13.79 → ingestr-0.13.81}/ingestr/src/facebook_ads/helpers.py

@@ -144,7 +144,7 @@ def execute_job(
             raise InsightsJobTimeout(
                 "facebook_insights",
                 pretty_error_message.format(
-                    job_id, insights_max_wait_to_finish_seconds // 60
+                    job_id, insights_max_wait_to_finish_seconds
                 ),
             )
 
@@ -229,3 +229,49 @@ def notify_on_token_expiration(access_token_expires_at: int = None) -> None:
             logger.error(
                 f"Access Token expires in {humanize.precisedelta(pendulum.now() - expires_at)}. Replace the token now!"
             )
+
+
+def parse_insights_table_to_source_kwargs(table: str) -> DictStrAny:
+    import typing
+
+    from ingestr.src.facebook_ads.settings import (
+        INSIGHTS_BREAKDOWNS_OPTIONS,
+        TInsightsBreakdownOptions,
+        TInsightsLevels,
+    )
+
+    parts = table.split(":")
+
+    source_kwargs = {}
+
+    breakdown_type = parts[1]
+
+    valid_breakdowns = list(typing.get_args(TInsightsBreakdownOptions))
+    if breakdown_type in valid_breakdowns:
+        dimensions = INSIGHTS_BREAKDOWNS_OPTIONS[breakdown_type]["breakdowns"]
+        fields = INSIGHTS_BREAKDOWNS_OPTIONS[breakdown_type]["fields"]
+        source_kwargs["dimensions"] = dimensions
+        source_kwargs["fields"] = fields
+    else:
+        dimensions = breakdown_type.split(",")
+        valid_levels = list(typing.get_args(TInsightsLevels))
+        level = None
+        for valid_level in reversed(valid_levels):
+            if valid_level in dimensions:
+                level = valid_level
+                dimensions.remove(valid_level)
+                break
+
+        source_kwargs["level"] = level
+        source_kwargs["dimensions"] = dimensions
+
+    # If custom metrics are provided, parse them
+    if len(parts) == 3:
+        fields = [f.strip() for f in parts[2].split(",") if f.strip()]
+        if not fields:
+            raise ValueError(
+                "Custom metrics must be provided after the second colon in format: facebook_insights:breakdown_type:metric1,metric2..."
+            )
+        source_kwargs["fields"] = fields
+
+    return source_kwargs

{ingestr-0.13.79 → ingestr-0.13.81}/ingestr/src/linear/__init__.py

@@ -3,7 +3,23 @@ from typing import Any, Dict, Iterable, Iterator
 import dlt
 import pendulum
 
-from .helpers import _normalize_issue, _normalize_team, _paginate
+from .helpers import _paginate, normalize_dictionaries
+
+
+def _get_date_range(updated_at, start_date):
+    """Extract current start and end dates from incremental state."""
+    if updated_at.last_value:
+        current_start_date = pendulum.parse(updated_at.last_value)
+    else:
+        current_start_date = pendulum.parse(start_date)
+
+    if updated_at.end_value:
+        current_end_date = pendulum.parse(updated_at.end_value)
+    else:
+        current_end_date = pendulum.now(tz="UTC")
+
+    return current_start_date, current_end_date
+
 
 ISSUES_QUERY = """
 query Issues($cursor: String) {
@@ -84,6 +100,25 @@ query Users($cursor: String) {
   }
 }
 """
+WORKFLOW_STATES_QUERY = """
+query WorkflowStates($cursor: String) {
+  workflowStates(first: 50, after: $cursor) {
+    nodes { 
+      archivedAt
+      color
+      createdAt
+      id
+      inheritedFrom { id }
+      name
+      position
+      team { id  }
+      type
+      updatedAt
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
 
 
 @dlt.source(name="linear", max_table_nesting=0)
@@ -102,20 +137,12 @@ def linear_source(
             range_end="closed",
         ),
     ) -> Iterator[Dict[str, Any]]:
-        if updated_at.last_value:
-            current_start_date = pendulum.parse(updated_at.last_value)
-        else:
-            current_start_date = pendulum.parse(start_date)
-
-        if updated_at.end_value:
-            current_end_date = pendulum.parse(updated_at.end_value)
-        else:
-            current_end_date = pendulum.now(tz="UTC")
+        current_start_date, current_end_date = _get_date_range(updated_at, start_date)
 
         for item in _paginate(api_key, ISSUES_QUERY, "issues"):
             if pendulum.parse(item["updatedAt"]) >= current_start_date:
                 if pendulum.parse(item["updatedAt"]) <= current_end_date:
-                    yield _normalize_issue(item)
+                    yield normalize_dictionaries(item)
 
     @dlt.resource(name="projects", primary_key="id", write_disposition="merge")
     def projects(
@@ -127,20 +154,12 @@ def linear_source(
             range_end="closed",
         ),
     ) -> Iterator[Dict[str, Any]]:
-        if updated_at.last_value:
-            current_start_date = pendulum.parse(updated_at.last_value)
-        else:
-            current_start_date = pendulum.parse(start_date)
-
-        if updated_at.end_value:
-            current_end_date = pendulum.parse(updated_at.end_value)
-        else:
-            current_end_date = pendulum.now(tz="UTC")
+        current_start_date, current_end_date = _get_date_range(updated_at, start_date)
 
         for item in _paginate(api_key, PROJECTS_QUERY, "projects"):
             if pendulum.parse(item["updatedAt"]) >= current_start_date:
                 if pendulum.parse(item["updatedAt"]) <= current_end_date:
-                    yield item
+                    yield normalize_dictionaries(item)
 
     @dlt.resource(name="teams", primary_key="id", write_disposition="merge")
     def teams(
@@ -153,21 +172,13 @@ def linear_source(
         ),
     ) -> Iterator[Dict[str, Any]]:
         print(start_date)
-        if updated_at.last_value:
-            current_start_date = pendulum.parse(updated_at.last_value)
-        else:
-            current_start_date = pendulum.parse(start_date)
+        current_start_date, current_end_date = _get_date_range(updated_at, start_date)
         print(current_start_date)
 
-        if updated_at.end_value:
-            current_end_date = pendulum.parse(updated_at.end_value)
-        else:
-            current_end_date = pendulum.now(tz="UTC")
-
         for item in _paginate(api_key, TEAMS_QUERY, "teams"):
             if pendulum.parse(item["updatedAt"]) >= current_start_date:
                 if pendulum.parse(item["updatedAt"]) <= current_end_date:
-                    yield _normalize_team(item)
+                    yield normalize_dictionaries(item)
 
     @dlt.resource(name="users", primary_key="id", write_disposition="merge")
     def users(
@@ -179,19 +190,28 @@ def linear_source(
             range_end="closed",
         ),
     ) -> Iterator[Dict[str, Any]]:
-        if updated_at.last_value:
-            current_start_date = pendulum.parse(updated_at.last_value)
-        else:
-            current_start_date = pendulum.parse(start_date)
-
-        if updated_at.end_value:
-            current_end_date = pendulum.parse(updated_at.end_value)
-        else:
-            current_end_date = pendulum.now(tz="UTC")
+        current_start_date, current_end_date = _get_date_range(updated_at, start_date)
 
         for item in _paginate(api_key, USERS_QUERY, "users"):
             if pendulum.parse(item["updatedAt"]) >= current_start_date:
                 if pendulum.parse(item["updatedAt"]) <= current_end_date:
-                    yield item
+                    yield normalize_dictionaries(item)
+
+    @dlt.resource(name="workflow_states", primary_key="id", write_disposition="merge")
+    def workflow_states(
+        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "updatedAt",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[Dict[str, Any]]:
+        current_start_date, current_end_date = _get_date_range(updated_at, start_date)
+
+        for item in _paginate(api_key, WORKFLOW_STATES_QUERY, "workflowStates"):
+            if pendulum.parse(item["updatedAt"]) >= current_start_date:
+                if pendulum.parse(item["updatedAt"]) <= current_end_date:
+                    yield normalize_dictionaries(item)
 
-    return issues, projects, teams, users
+    return [issues, projects, teams, users, workflow_states]

ingestr-0.13.81/ingestr/src/linear/helpers.py

@@ -0,0 +1,53 @@
+from typing import Any, Dict, Iterator, Optional
+
+import requests
+
+LINEAR_GRAPHQL_ENDPOINT = "https://api.linear.app/graphql"
+
+
+def _graphql(
+    api_key: str, query: str, variables: Optional[Dict[str, Any]] = None
+) -> Dict[str, Any]:
+    headers = {"Authorization": api_key, "Content-Type": "application/json"}
+    response = requests.post(
+        LINEAR_GRAPHQL_ENDPOINT,
+        json={"query": query, "variables": variables or {}},
+        headers=headers,
+    )
+    response.raise_for_status()
+    payload = response.json()
+    if "errors" in payload:
+        raise ValueError(str(payload["errors"]))
+    return payload["data"]
+
+
+def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
+    cursor: Optional[str] = None
+    while True:
+        data = _graphql(api_key, query, {"cursor": cursor})[root]
+        for item in data["nodes"]:
+            yield item
+        if not data["pageInfo"]["hasNextPage"]:
+            break
+        cursor = data["pageInfo"]["endCursor"]
+
+
+def normalize_dictionaries(item: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Automatically normalize dictionary fields by detecting their structure:
+    - Convert nested objects with 'id' field to {field_name}_id
+    - Convert objects with 'nodes' field to arrays
+    """
+    normalized_item = item.copy()
+
+    for key, value in list(normalized_item.items()):
+        if isinstance(value, dict):
+            # If the dict has an 'id' field, replace with {key}_id
+            if "id" in value:
+                normalized_item[f"{key}_id"] = value["id"]
+                del normalized_item[key]
+            # If the dict has 'nodes' field, extract the nodes array
+            elif "nodes" in value:
+                normalized_item[key] = value["nodes"]
+
+    return normalized_item