ingestr 0.13.80__py3-none-any.whl → 0.13.82__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.80"
+version = "v0.13.82"

ingestr/src/facebook_ads/__init__.py

@@ -22,12 +22,8 @@ from .settings import (
     DEFAULT_ADCREATIVE_FIELDS,
     DEFAULT_ADSET_FIELDS,
     DEFAULT_CAMPAIGN_FIELDS,
-    DEFAULT_INSIGHT_FIELDS,
     DEFAULT_LEAD_FIELDS,
     INSIGHT_FIELDS_TYPES,
-    INSIGHTS_BREAKDOWNS_OPTIONS,
-    INVALID_INSIGHTS_FIELDS,
-    TInsightsBreakdownOptions,
     TInsightsLevels,
 )
 
@@ -105,10 +101,9 @@ def facebook_insights_source(
     account_id: str = dlt.config.value,
     access_token: str = dlt.secrets.value,
     initial_load_past_days: int = 1,
-    fields: Sequence[str] = DEFAULT_INSIGHT_FIELDS,
-    attribution_window_days_lag: int = 7,
+    dimensions: Sequence[str] = None,
+    fields: Sequence[str] = None,
     time_increment_days: int = 1,
-    breakdowns: TInsightsBreakdownOptions = "ads_insights",
     action_breakdowns: Sequence[str] = ALL_ACTION_BREAKDOWNS,
     level: TInsightsLevels = "ad",
     action_attribution_windows: Sequence[str] = ALL_ACTION_ATTRIBUTION_WINDOWS,
@@ -155,6 +150,11 @@ def facebook_insights_source(
     if start_date is None:
         start_date = pendulum.today().subtract(days=initial_load_past_days)
 
+    if dimensions is None:
+        dimensions = []
+    if fields is None:
+        fields = []
+
     columns = {}
     for field in fields:
         if field in INSIGHT_FIELDS_TYPES:
@@ -187,15 +187,9 @@ def facebook_insights_source(
             query = {
                 "level": level,
                 "action_breakdowns": list(action_breakdowns),
-                "breakdowns": list(
-                    INSIGHTS_BREAKDOWNS_OPTIONS[breakdowns]["breakdowns"]
-                ),
+                "breakdowns": dimensions,
                 "limit": batch_size,
-                "fields": list(
-                    set(fields)
-                    .union(INSIGHTS_BREAKDOWNS_OPTIONS[breakdowns]["fields"])
-                    .difference(INVALID_INSIGHTS_FIELDS)
-                ),
+                "fields": fields,
                 "time_increment": time_increment_days,
                 "action_attribution_windows": list(action_attribution_windows),
                 "time_ranges": [

ingestr/src/facebook_ads/helpers.py

@@ -229,3 +229,49 @@ def notify_on_token_expiration(access_token_expires_at: int = None) -> None:
             logger.error(
                 f"Access Token expires in {humanize.precisedelta(pendulum.now() - expires_at)}. Replace the token now!"
             )
+
+
+def parse_insights_table_to_source_kwargs(table: str) -> DictStrAny:
+    import typing
+
+    from ingestr.src.facebook_ads.settings import (
+        INSIGHTS_BREAKDOWNS_OPTIONS,
+        TInsightsBreakdownOptions,
+        TInsightsLevels,
+    )
+
+    parts = table.split(":")
+
+    source_kwargs = {}
+
+    breakdown_type = parts[1]
+
+    valid_breakdowns = list(typing.get_args(TInsightsBreakdownOptions))
+    if breakdown_type in valid_breakdowns:
+        dimensions = INSIGHTS_BREAKDOWNS_OPTIONS[breakdown_type]["breakdowns"]
+        fields = INSIGHTS_BREAKDOWNS_OPTIONS[breakdown_type]["fields"]
+        source_kwargs["dimensions"] = dimensions
+        source_kwargs["fields"] = fields
+    else:
+        dimensions = breakdown_type.split(",")
+        valid_levels = list(typing.get_args(TInsightsLevels))
+        level = None
+        for valid_level in reversed(valid_levels):
+            if valid_level in dimensions:
+                level = valid_level
+                dimensions.remove(valid_level)
+                break
+
+        source_kwargs["level"] = level
+        source_kwargs["dimensions"] = dimensions
+
+    # If custom metrics are provided, parse them
+    if len(parts) == 3:
+        fields = [f.strip() for f in parts[2].split(",") if f.strip()]
+        if not fields:
+            raise ValueError(
+                "Custom metrics must be provided after the second colon in format: facebook_insights:breakdown_type:metric1,metric2..."
+            )
+        source_kwargs["fields"] = fields
+
+    return source_kwargs

ingestr/src/linear/__init__.py

@@ -2,24 +2,11 @@ from typing import Any, Dict, Iterable, Iterator
 
 import dlt
 import pendulum
+import requests
 
-from .helpers import _paginate, normalize_dictionaries
+from .helpers import _graphql, normalize_dictionaries, _get_date_range, _create_paginated_resource
 
 
-def _get_date_range(updated_at, start_date):
-    """Extract current start and end dates from incremental state."""
-    if updated_at.last_value:
-        current_start_date = pendulum.parse(updated_at.last_value)
-    else:
-        current_start_date = pendulum.parse(start_date)
-
-    if updated_at.end_value:
-        current_end_date = pendulum.parse(updated_at.end_value)
-    else:
-        current_end_date = pendulum.now(tz="UTC")
-    
-    return current_start_date, current_end_date
-
 ISSUES_QUERY = """
 query Issues($cursor: String) {
   issues(first: 50, after: $cursor) {
@@ -29,55 +16,409 @@ query Issues($cursor: String) {
       description
       createdAt
       updatedAt
-      creator { id }
-      assignee { id}
-      state { id}
-      labels { nodes { id } }
-      cycle { id}
-      project { id }
-      subtasks: children { nodes { id title } }
-      comments(first: 250) { nodes { id body } }
+      archivedAt
+      addedToCycleAt
+      autoArchivedAt
+      autoClosedAt
+      boardOrder
+      branchName
+      canceledAt
+      completedAt
+      customerTicketCount
+      descriptionState
+      dueDate
+      estimate
+      identifier
+      integrationSourceType
+      labelIds
+      number
+      previousIdentifiers
       priority
-      attachments { nodes { id } }
-      subscribers { nodes { id } }
+      priorityLabel
+      prioritySortOrder
+      reactionData
+      slaBreachesAt
+      slaHighRiskAt
+      slaMediumRiskAt
+      slaStartedAt
+      slaType
+      snoozedUntilAt
+      sortOrder
+      startedAt
+      startedTriageAt
+      subIssueSortOrder
+      triagedAt
+      url
+      
+      creator { id }
+      assignee { id }
+      botActor { id name type }
+      cycle { id }
+      delegate { id }
+      externalUserCreator { id }
+      favorite { id }
+      lastAppliedTemplate { id }
+      parent { id }
+      projectMilestone { id }
+      recurringIssueTemplate { id }
+      snoozedBy { id }
+      sourceComment { id }
+      state { id }
+      
+      labels(first: 250) { 
+        nodes { 
+          id 
+        } 
+      }
     }
     pageInfo { hasNextPage endCursor }
   }
 }
 """
 
-PROJECTS_QUERY = """
-query Projects($cursor: String) {
-  projects(first: 50, after: $cursor) {
+ATTACHMENTS_QUERY = """
+query Attachments($cursor: String) {
+  attachments(first: 50, after: $cursor) {
     nodes {
       id
-      name
+      archivedAt
+      bodyData
+      createdAt
+      groupBySource
+      metadata
+      sourceType
+      subtitle
+      title
+      updatedAt
+      url
+      
+      creator { id }
+      externalUserCreator { id }
+      issue { id }
+      originalIssue { id }
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+COMMENTS_QUERY = """
+query Comments($cursor: String) {
+  comments(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
+      body
+      bodyData
+      createdAt
+      editedAt
+      quotedText
+      reactionData 
+      resolvedAt
+      threadSummary
+      updatedAt
+      url
+      
+      botActor { id  }
+      documentContent { id }
+      externalThread { id }
+      externalUser { id }
+      initiativeUpdate { id }
+      issue { id }
+      parent { id }
+      post { id }
+      projectUpdate { id }
+      resolvingComment { id }
+      resolvingUser { id }
+      user { id }
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+CYCLES_QUERY = """
+query Cycles($cursor: String) {
+  cycles(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
+      autoArchivedAt
+      completedAt
+      completedIssueCountHistory
+      completedScopeHistory
+      createdAt
       description
+      endsAt
+      inProgressScopeHistory
+      issueCountHistory
+      name
+      number
+      progress
+      scopeHistory
+      startsAt
+      updatedAt
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+DOCUMENTS_QUERY = """
+query Documents($cursor: String) {
+  documents(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
+      color
       createdAt
+      icon
+      slugId
+      title
       updatedAt
+      
+      creator { id }
+      updatedBy { id }
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+EXTERNAL_USERS_QUERY = """
+query ExternalUsers($cursor: String) {
+  externalUsers(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
+      avatarUrl
+      createdAt
+      displayName
+      email
+      lastSeen
+      name
+      updatedAt
+      
+      organization { id }
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+INITIATIVES_QUERY = """
+query Initiatives($cursor: String) {
+  initiatives(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
+      color
+      completedAt
+      content
+      createdAt
+      description
+      frequencyResolution
       health
-      priority
+      healthUpdatedAt
+      icon
+      name
+      slugId
+      sortOrder
+      startedAt
+      status
       targetDate
-      lead { id }
+      targetDateResolution
+      trashed
+      updateReminderFrequency
+      updateReminderFrequencyInWeeks
+      updateRemindersDay
+      updateRemindersHour
+      updatedAt
+      
+      creator { id }
+      documentContent { id }
+      integrationsSettings { id }
+      lastUpdate { id }
+      organization { id }
+      owner { id }
+      parentInitiative { id }
     }
     pageInfo { hasNextPage endCursor }
   }
 }
 """
 
-TEAMS_QUERY = """
-query Teams($cursor: String) {
-  teams(first: 50, after: $cursor) {
+
+
+INITIATIVE_TO_PROJECTS_QUERY = """
+query InitiativeToProjects($cursor: String) {
+  initiativeToProjects(first: 50, after: $cursor) {
     nodes {
       id
+      archivedAt
+      createdAt
+      sortOrder
+      updatedAt
+      
+      initiative { id }
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+PROJECT_MILESTONES_QUERY = """
+query ProjectMilestones($cursor: String) {
+  projectMilestones(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
+      createdAt
+      currentProgress
+      description
+      descriptionState
       name
-      key
+      progress
+      progressHistory
+      sortOrder
+      status
+      targetDate
+      updatedAt
+      
+      documentContent { id }
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+PROJECT_STATUSES_QUERY = """
+query ProjectStatuses($cursor: String) {
+  projectStatuses(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
+      color
+      createdAt
       description
+      indefinite
+      name
+      position
+      type
       updatedAt
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+INTEGRATIONS_QUERY = """
+query Integrations($cursor: String) {
+  integrations(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
       createdAt
-      memberships { nodes { id } }
-      members { nodes { id } }
-      projects { nodes { id } }
+      service
+      updatedAt
+      
+      creator { id }
+      organization { id }
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+
+LABELS_QUERY = """
+query IssueLabels($cursor: String) {
+  issueLabels(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
+      color
+      createdAt
+      description
+      name
+      updatedAt
+      
+      creator { id }
+      organization { id }
+      parent { id }
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+
+ORGANIZATION_QUERY = """
+query Organization {
+  viewer {
+    organization {
+      id
+      name
+      createdAt
+      updatedAt
+      archivedAt
+      logoUrl
+      allowMembersToInvite
+      allowedAuthServices
+      createdIssueCount
+      customerCount
+      customersEnabled
+      deletionRequestedAt
+      gitBranchFormat
+      gitLinkbackMessagesEnabled
+      gitPublicLinkbackMessagesEnabled
+      logoUrl
+      periodUploadVolume
+      previousUrlKeys
+      roadmapEnabled
+      samlEnabled
+      scimEnabled
+    }
+  }
+}
+"""
+
+
+PROJECT_UPDATES_QUERY = """
+query ProjectUpdates($cursor: String) {
+  projectUpdates(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
+      body
+      bodyData
+      createdAt
+      diffMarkdown
+      health
+      updatedAt
+      url
+      
+      user { id }
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+
+
+TEAM_MEMBERSHIPS_QUERY = """
+query TeamMemberships($cursor: String) {
+  teamMemberships(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
+      createdAt
+      owner
+      sortOrder
+      updatedAt
+      
+      user { id }
     }
     pageInfo { hasNextPage endCursor }
   }
@@ -89,11 +430,29 @@ query Users($cursor: String) {
   users(first: 50, after: $cursor) {
     nodes {
       id
-      name
+      active
+      admin
+      archivedAt
+      avatarUrl
+      calendarHash
+      createdAt
+      createdIssueCount
+      description
+      disableReason
       displayName
       email
-      createdAt
+      guest
+      inviteHash
+      lastSeen
+      name
+      statusEmoji
+      statusLabel
+      statusUntilAt
+      timezone
       updatedAt
+      url
+      
+      organization { id }
     }
     pageInfo { hasNextPage endCursor }
   }
@@ -102,15 +461,14 @@ query Users($cursor: String) {
 WORKFLOW_STATES_QUERY = """
 query WorkflowStates($cursor: String) {
   workflowStates(first: 50, after: $cursor) {
-    nodes { 
+    nodes {
+      id
       archivedAt
       color
       createdAt
-      id
-      inheritedFrom { id }
+      description
       name
       position
-      team { id  }
       type
       updatedAt
     }
@@ -118,6 +476,48 @@ query WorkflowStates($cursor: String) {
   }
 }
 """
+PROJECTS_QUERY = """
+query Projects($cursor: String) {
+  projects(first: 50, after: $cursor) {
+    nodes {
+      id
+      name
+      description
+      createdAt
+      updatedAt
+      health
+      priority
+      targetDate
+      lead { id }
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+
+# Paginated resources configuration
+PAGINATED_RESOURCES = [
+    ("issues", ISSUES_QUERY, "issues"),
+    ("users", USERS_QUERY, "users"),
+    ("workflow_states", WORKFLOW_STATES_QUERY, "workflowStates"),
+    ("cycles", CYCLES_QUERY, "cycles"),
+    ("attachments", ATTACHMENTS_QUERY, "attachments"),
+    ("comments", COMMENTS_QUERY, "comments"),
+    ("documents", DOCUMENTS_QUERY, "documents"),
+    ("external_users", EXTERNAL_USERS_QUERY, "externalUsers"),
+    ("initiative", INITIATIVES_QUERY, "initiatives"),
+    ("integrations", INTEGRATIONS_QUERY, "integrations"),
+    ("labels", LABELS_QUERY, "issueLabels"),
+    ("project_updates", PROJECT_UPDATES_QUERY, "projectUpdates"),
+    ("team_memberships", TEAM_MEMBERSHIPS_QUERY, "teamMemberships"),
+    ("initiative_to_project", INITIATIVE_TO_PROJECTS_QUERY, "initiativeToProjects"),
+    ("project_milestone", PROJECT_MILESTONES_QUERY, "projectMilestones"),
+    ("project_status", PROJECT_STATUSES_QUERY, "projectStatuses"),
+    ("projects", PROJECTS_QUERY, "projects"),
+]
+
+
 
 @dlt.source(name="linear", max_table_nesting=0)
 def linear_source(
@@ -125,25 +525,9 @@ def linear_source(
     start_date: pendulum.DateTime,
     end_date: pendulum.DateTime | None = None,
 ) -> Iterable[dlt.sources.DltResource]:
-    @dlt.resource(name="issues", primary_key="id", write_disposition="merge")
-    def issues(
-        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
-            "updatedAt",
-            initial_value=start_date.isoformat(),
-            end_value=end_date.isoformat() if end_date else None,
-            range_start="closed",
-            range_end="closed",
-        ),
-    ) -> Iterator[Dict[str, Any]]:
-        current_start_date, current_end_date = _get_date_range(updated_at, start_date)
-
-        for item in _paginate(api_key, ISSUES_QUERY, "issues"):
-            if pendulum.parse(item["updatedAt"]) >= current_start_date:
-                if pendulum.parse(item["updatedAt"]) <= current_end_date:
-                    yield normalize_dictionaries(item)
 
-    @dlt.resource(name="projects", primary_key="id", write_disposition="merge")
-    def projects(
+    @dlt.resource(name="organization", primary_key="id", write_disposition="merge")
+    def organization(
         updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
             "updatedAt",
             initial_value=start_date.isoformat(),
@@ -154,62 +538,20 @@ def linear_source(
     ) -> Iterator[Dict[str, Any]]:
         current_start_date, current_end_date = _get_date_range(updated_at, start_date)
 
-        for item in _paginate(api_key, PROJECTS_QUERY, "projects"):
-            if pendulum.parse(item["updatedAt"]) >= current_start_date:
+        data = _graphql(api_key, ORGANIZATION_QUERY)
+        if "viewer" in data and "organization" in data["viewer"]:
+            item = data["viewer"]["organization"]
+            if item and pendulum.parse(item["updatedAt"]) >= current_start_date:
                 if pendulum.parse(item["updatedAt"]) <= current_end_date:
                     yield normalize_dictionaries(item)
 
-    @dlt.resource(name="teams", primary_key="id", write_disposition="merge")
-    def teams(
-        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
-            "updatedAt",
-            initial_value=start_date.isoformat(),
-            end_value=end_date.isoformat() if end_date else None,
-            range_start="closed",
-            range_end="closed",
-        ),
-    ) -> Iterator[Dict[str, Any]]:
-        print(start_date)
-        current_start_date, current_end_date = _get_date_range(updated_at, start_date)
-        print(current_start_date)
-
-        for item in _paginate(api_key, TEAMS_QUERY, "teams"):
-            if pendulum.parse(item["updatedAt"]) >= current_start_date:
-                if pendulum.parse(item["updatedAt"]) <= current_end_date:
-                    yield normalize_dictionaries(item)
-
-    @dlt.resource(name="users", primary_key="id", write_disposition="merge")
-    def users(
-        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
-            "updatedAt",
-            initial_value=start_date.isoformat(),
-            end_value=end_date.isoformat() if end_date else None,
-            range_start="closed",
-            range_end="closed",
-        ),
-    ) -> Iterator[Dict[str, Any]]:
-        current_start_date, current_end_date = _get_date_range(updated_at, start_date)
-
-        for item in _paginate(api_key, USERS_QUERY, "users"):
-            if pendulum.parse(item["updatedAt"]) >= current_start_date:
-                if pendulum.parse(item["updatedAt"]) <= current_end_date:
-                    yield normalize_dictionaries(item)
-
-    @dlt.resource(name="workflow_states", primary_key="id", write_disposition="merge")
-    def workflow_states(
-        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
-            "updatedAt",
-            initial_value=start_date.isoformat(),
-            end_value=end_date.isoformat() if end_date else None,
-            range_start="closed",
-            range_end="closed",
-        ),
-    ) -> Iterator[Dict[str, Any]]:
-        current_start_date, current_end_date = _get_date_range(updated_at, start_date)
-
-        for item in _paginate(api_key, WORKFLOW_STATES_QUERY, "workflowStates"):
-            if pendulum.parse(item["updatedAt"]) >= current_start_date:
-                if pendulum.parse(item["updatedAt"]) <= current_end_date:
-                    yield normalize_dictionaries(item)
-    return [issues, projects, teams, users, workflow_states]
-
+    # Create paginated resources dynamically
+    paginated_resources = [
+        _create_paginated_resource(resource_name, query, query_field, api_key, start_date, end_date)
+        for resource_name, query, query_field in PAGINATED_RESOURCES
+    ]
+    
+    return [
+        *paginated_resources,
+        organization,
+    ]

ingestr/src/linear/helpers.py

@@ -1,6 +1,8 @@
 from typing import Any, Dict, Iterator, Optional
 
 import requests
+import pendulum
+import dlt
 
 LINEAR_GRAPHQL_ENDPOINT = "https://api.linear.app/graphql"
 
@@ -34,22 +36,66 @@ def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
 
 
 
+def _get_date_range(updated_at, start_date):
+    """Extract current start and end dates from incremental state."""
+    if updated_at.last_value:
+        current_start_date = pendulum.parse(updated_at.last_value)
+    else:
+        current_start_date = pendulum.parse(start_date)
+
+    if updated_at.end_value:
+        current_end_date = pendulum.parse(updated_at.end_value)
+    else:
+        current_end_date = pendulum.now(tz="UTC")
+    
+    return current_start_date, current_end_date
+
+
+def _paginated_resource(api_key: str, query: str, query_field: str, updated_at, start_date) -> Iterator[Dict[str, Any]]:
+    """Helper function for paginated resources with date filtering."""
+    current_start_date, current_end_date = _get_date_range(updated_at, start_date)
+
+    for item in _paginate(api_key, query, query_field):
+        if pendulum.parse(item["updatedAt"]) >= current_start_date:
+            if pendulum.parse(item["updatedAt"]) <= current_end_date:
+                yield normalize_dictionaries(item)
+
+
+def _create_paginated_resource(resource_name: str, query: str, query_field: str, api_key: str, start_date, end_date = None):
+    """Factory function to create paginated resources dynamically."""
+    @dlt.resource(name=resource_name, primary_key="id", write_disposition="merge")
+    def paginated_resource(
+        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "updatedAt",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[Dict[str, Any]]:
+        for item in _paginated_resource(api_key, query, query_field, updated_at, start_date):
+            yield normalize_dictionaries(item)
+    
+    return paginated_resource
+
+
 def normalize_dictionaries(item: Dict[str, Any]) -> Dict[str, Any]:
     """
     Automatically normalize dictionary fields by detecting their structure:
     - Convert nested objects with 'id' field to {field_name}_id
     - Convert objects with 'nodes' field to arrays
+ 
     """
     normalized_item = item.copy()
-    
+
     for key, value in list(normalized_item.items()):
         if isinstance(value, dict):
             # If the dict has an 'id' field, replace with {key}_id
-            if 'id' in value:
-                normalized_item[f"{key}_id"] = value['id']
+            if "id" in value:
+                normalized_item[f"{key}_id"] = value["id"]
                 del normalized_item[key]
             # If the dict has 'nodes' field, extract the nodes array
-            elif 'nodes' in value:
-                normalized_item[key] = value['nodes']
-    
+            elif "nodes" in value:
+                normalized_item[key] = value["nodes"]
+
     return normalized_item

ingestr/src/sources.py

@@ -1056,33 +1056,16 @@ class FacebookAdsSource:
             # Validate breakdown type against available options from settings
             import typing
 
-            from ingestr.src.facebook_ads.settings import TInsightsBreakdownOptions
-
-            # Get valid breakdown options from the type definition
-            valid_breakdowns = list(typing.get_args(TInsightsBreakdownOptions))
-
-            if breakdown_type not in valid_breakdowns:
-                raise ValueError(
-                    f"Invalid breakdown type '{breakdown_type}'. Valid options: {', '.join(valid_breakdowns)}"
-                )
+            from ingestr.src.facebook_ads.helpers import parse_insights_table_to_source_kwargs
 
             source_kwargs = {
                 "access_token": access_token[0],
                 "account_id": account_id[0],
                 "start_date": kwargs.get("interval_start"),
                 "end_date": kwargs.get("interval_end"),
-                "breakdowns": breakdown_type,
             }
 
-            # If custom metrics are provided, parse them
-            if len(parts) == 3:
-                fields = [f.strip() for f in parts[2].split(",") if f.strip()]
-                if not fields:
-                    raise ValueError(
-                        "Custom metrics must be provided after the second colon in format: facebook_insights:breakdown_type:metric1,metric2..."
-                    )
-                source_kwargs["fields"] = fields
-
+            source_kwargs.update(parse_insights_table_to_source_kwargs(table))
             return facebook_insights_source(**source_kwargs).with_resources(
                 "facebook_insights"
             )
@@ -3257,7 +3240,27 @@ class LinearSource:
         if api_key is None:
             raise MissingValueError("api_key", "Linear")
 
-        if table not in ["issues", "projects", "teams", "users", "workflow_states"]:
+        if table not in [
+            "issues", 
+            "projects", 
+            "teams", 
+            "users", 
+            "workflow_states", 
+            "cycles",
+            "attachments",
+            "comments",
+            "documents",
+            "external_users",
+            "initiative",
+            "integrations",
+            "labels",
+            "organization",
+            "project_updates",
+            "team_memberships",
+            "initiative_to_project",
+            "project_milestone",
+            "project_status",
+        ]:
             raise UnsupportedResourceError(table, "Linear")
 
         start_date = kwargs.get("interval_start")

{ingestr-0.13.80.dist-info → ingestr-0.13.82.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.80
+Version: 0.13.82
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.80.dist-info → ingestr-0.13.82.dist-info}/RECORD

@@ -2,7 +2,7 @@ ingestr/conftest.py,sha256=OE2yxeTCosS9CUFVuqNypm-2ftYvVBeeq7egm3878cI,1981
 ingestr/main.py,sha256=qoWHNcHh0-xVnyQxbQ-SKuTxPb1RNV3ENkCpqO7CLrk,26694
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
-ingestr/src/buildinfo.py,sha256=K8l-cnsoOmTtSy1GdxNMRLZxsjHoYiJsCc5rvXeCpHE,21
+ingestr/src/buildinfo.py,sha256=S1Cgqn9Xc0avTk0r_rJ8fMXucqD7c5O68FsKjSWj2Pc,21
 ingestr/src/destinations.py,sha256=M2Yni6wiWcrvZ8EPJemidqxN156l0rehgCc7xuil7mo,22840
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
 ingestr/src/factory.py,sha256=rF5Ry4o4t8KulSPBtrd7ZKCI_0TH1DAetG0zs9H7oik,6792
@@ -11,7 +11,7 @@ ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,73
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=guwCdKQDvT2XMYeR2O3nJ9kZ-wLCSDpOex4UH0luG5k,119966
+ingestr/src/sources.py,sha256=Yaej1Iy75JPOccw6jYL6kbhzYFq33t4YelGT-NiAA5I,119661
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -41,9 +41,9 @@ ingestr/src/clickup/helpers.py,sha256=RzDKMUAHccuDhocIQ2ToBXfCERo8CBJqA3t-IPltBC
 ingestr/src/collector/spinner.py,sha256=_ZUqF5MI43hVIULdjF5s5mrAZbhEFXaiWirQmrv3Yk4,1201
 ingestr/src/dynamodb/__init__.py,sha256=swhxkeYBbJ35jn1IghCtvYWT2BM33KynVCh_oR4z28A,2264
 ingestr/src/elasticsearch/__init__.py,sha256=m-q93HgUmTwGDUwHOjHawstWL06TC3WIX3H05szybrY,2556
-ingestr/src/facebook_ads/__init__.py,sha256=bX6lnf0LxIcOyZHDVA9FL5iKhgnQ0f5Hfma4eXcQuIk,10094
+ingestr/src/facebook_ads/__init__.py,sha256=15GiovITANe0al5MI6WWLdl3LDmdBd1YpkUWBV3g6bk,9715
 ingestr/src/facebook_ads/exceptions.py,sha256=4Nlbc0Mv3i5g-9AoyT-n1PIa8IDi3VCTfEAzholx4Wc,115
-ingestr/src/facebook_ads/helpers.py,sha256=Oh9-LepxxBRnPXQZMbbNOSbxg9T8a4nmiLSt22GPt6E,8233
+ingestr/src/facebook_ads/helpers.py,sha256=c-WG008yU_zIdhFwljtqE2jfjVYuaVoNKldxcnJN3U4,9761
 ingestr/src/facebook_ads/settings.py,sha256=Bsic8RcmH-NfEZ7r_NGospTCmwISK9XaMT5y2NZirtg,4938
 ingestr/src/facebook_ads/utils.py,sha256=ES2ylPoW3j3fjp6OMUgp21n1cG1OktXsmWWMk5vBW_I,1590
 ingestr/src/filesystem/__init__.py,sha256=zkIwbRr0ir0EUdniI25p2zGiVc-7M9EmR351AjNb0eA,4163
@@ -85,8 +85,8 @@ ingestr/src/kinesis/helpers.py,sha256=SO2cFmWNGcykUYmjHdfxWsOQSkLQXyhFtfWnkcUOM0
 ingestr/src/klaviyo/__init__.py,sha256=o_noUgbxLk36s4f9W56_ibPorF0n7kVapPUlV0p-jfA,7875
 ingestr/src/klaviyo/client.py,sha256=tPj79ia7AW0ZOJhzlKNPCliGbdojRNwUFp8HvB2ym5s,7434
 ingestr/src/klaviyo/helpers.py,sha256=_i-SHffhv25feLDcjy6Blj1UxYLISCwVCMgGtrlnYHk,496
-ingestr/src/linear/__init__.py,sha256=XWgWiDJi87OFHnsOjTq4ZSAdtMcPdplBYC8fJM-6dMA,6607
-ingestr/src/linear/helpers.py,sha256=y8XhEDBVnxMmSzzyrS0_RnPwtNJIRuKM4Kw3wW9p6UM,1796
+ingestr/src/linear/__init__.py,sha256=G2x4HaRl8WWReDJ5HElnNTfVIybdztrIB3acl1ickyo,11003
+ingestr/src/linear/helpers.py,sha256=pPqtJrY3D3U2IaqTsD6ChvwMLYiUVtmQ8wTvetz5U7A,3640
 ingestr/src/linkedin_ads/__init__.py,sha256=CAPWFyV24loziiphbLmODxZUXZJwm4JxlFkr56q0jfo,1855
 ingestr/src/linkedin_ads/dimension_time_enum.py,sha256=EmHRdkFyTAfo4chGjThrwqffWJxmAadZMbpTvf0xkQc,198
 ingestr/src/linkedin_ads/helpers.py,sha256=eUWudRVlXl4kqIhfXQ1eVsUpZwJn7UFqKSpnbLfxzds,4498
@@ -151,8 +151,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
-ingestr-0.13.80.dist-info/METADATA,sha256=HF4xO0hXDV_gn389MIuX_-GhahKPaz8GOMIMt1If4x0,15182
-ingestr-0.13.80.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.80.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.80.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.80.dist-info/RECORD,,
+ingestr-0.13.82.dist-info/METADATA,sha256=4edZaTP7apukcZRUY4J00G99O3d7Qxz37csXXCNhaww,15182
+ingestr-0.13.82.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.82.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.82.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.82.dist-info/RECORD,,