ingestr 0.13.79__tar.gz → 0.13.80__tar.gz

{ingestr-0.13.79 → ingestr-0.13.80}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.79
+Version: 0.13.80
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

ingestr-0.13.80/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.80"

{ingestr-0.13.79 → ingestr-0.13.80}/ingestr/src/facebook_ads/__init__.py

@@ -117,6 +117,9 @@ def facebook_insights_source(
     app_api_version: str = None,
     start_date: pendulum.DateTime | None = None,
     end_date: pendulum.DateTime | None = None,
+    insights_max_wait_to_finish_seconds: int = 60 * 60 * 4,
+    insights_max_wait_to_start_seconds: int = 60 * 30,
+    insights_max_async_sleep_seconds: int = 20,
 ) -> DltResource:
     """Incrementally loads insight reports with defined granularity level, fields, breakdowns etc.
 
@@ -206,7 +209,9 @@ def facebook_insights_source(
             }
             job = execute_job(
                 account.get_insights(params=query, is_async=True),
-                insights_max_async_sleep_seconds=20,
+                insights_max_async_sleep_seconds=insights_max_async_sleep_seconds,
+                insights_max_wait_to_finish_seconds=insights_max_wait_to_finish_seconds,
+                insights_max_wait_to_start_seconds=insights_max_wait_to_start_seconds,
             )
             output = list(map(process_report_item, job.get_result()))
             yield output

{ingestr-0.13.79 → ingestr-0.13.80}/ingestr/src/facebook_ads/helpers.py

@@ -144,7 +144,7 @@ def execute_job(
             raise InsightsJobTimeout(
                 "facebook_insights",
                 pretty_error_message.format(
-                    job_id, insights_max_wait_to_finish_seconds // 60
+                    job_id, insights_max_wait_to_finish_seconds
                 ),
             )
 

{ingestr-0.13.79 → ingestr-0.13.80}/ingestr/src/linear/__init__.py

@@ -3,7 +3,22 @@ from typing import Any, Dict, Iterable, Iterator
 import dlt
 import pendulum
 
-from .helpers import _normalize_issue, _normalize_team, _paginate
+from .helpers import _paginate, normalize_dictionaries
+
+
+def _get_date_range(updated_at, start_date):
+    """Extract current start and end dates from incremental state."""
+    if updated_at.last_value:
+        current_start_date = pendulum.parse(updated_at.last_value)
+    else:
+        current_start_date = pendulum.parse(start_date)
+
+    if updated_at.end_value:
+        current_end_date = pendulum.parse(updated_at.end_value)
+    else:
+        current_end_date = pendulum.now(tz="UTC")
+    
+    return current_start_date, current_end_date
 
 ISSUES_QUERY = """
 query Issues($cursor: String) {
@@ -84,7 +99,25 @@ query Users($cursor: String) {
   }
 }
 """
-
+WORKFLOW_STATES_QUERY = """
+query WorkflowStates($cursor: String) {
+  workflowStates(first: 50, after: $cursor) {
+    nodes { 
+      archivedAt
+      color
+      createdAt
+      id
+      inheritedFrom { id }
+      name
+      position
+      team { id  }
+      type
+      updatedAt
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
 
 @dlt.source(name="linear", max_table_nesting=0)
 def linear_source(
@@ -102,20 +135,12 @@ def linear_source(
             range_end="closed",
         ),
     ) -> Iterator[Dict[str, Any]]:
-        if updated_at.last_value:
-            current_start_date = pendulum.parse(updated_at.last_value)
-        else:
-            current_start_date = pendulum.parse(start_date)
-
-        if updated_at.end_value:
-            current_end_date = pendulum.parse(updated_at.end_value)
-        else:
-            current_end_date = pendulum.now(tz="UTC")
+        current_start_date, current_end_date = _get_date_range(updated_at, start_date)
 
         for item in _paginate(api_key, ISSUES_QUERY, "issues"):
             if pendulum.parse(item["updatedAt"]) >= current_start_date:
                 if pendulum.parse(item["updatedAt"]) <= current_end_date:
-                    yield _normalize_issue(item)
+                    yield normalize_dictionaries(item)
 
     @dlt.resource(name="projects", primary_key="id", write_disposition="merge")
     def projects(
@@ -127,20 +152,12 @@ def linear_source(
             range_end="closed",
         ),
     ) -> Iterator[Dict[str, Any]]:
-        if updated_at.last_value:
-            current_start_date = pendulum.parse(updated_at.last_value)
-        else:
-            current_start_date = pendulum.parse(start_date)
-
-        if updated_at.end_value:
-            current_end_date = pendulum.parse(updated_at.end_value)
-        else:
-            current_end_date = pendulum.now(tz="UTC")
+        current_start_date, current_end_date = _get_date_range(updated_at, start_date)
 
         for item in _paginate(api_key, PROJECTS_QUERY, "projects"):
             if pendulum.parse(item["updatedAt"]) >= current_start_date:
                 if pendulum.parse(item["updatedAt"]) <= current_end_date:
-                    yield item
+                    yield normalize_dictionaries(item)
 
     @dlt.resource(name="teams", primary_key="id", write_disposition="merge")
     def teams(
@@ -153,21 +170,13 @@ def linear_source(
         ),
     ) -> Iterator[Dict[str, Any]]:
         print(start_date)
-        if updated_at.last_value:
-            current_start_date = pendulum.parse(updated_at.last_value)
-        else:
-            current_start_date = pendulum.parse(start_date)
+        current_start_date, current_end_date = _get_date_range(updated_at, start_date)
         print(current_start_date)
 
-        if updated_at.end_value:
-            current_end_date = pendulum.parse(updated_at.end_value)
-        else:
-            current_end_date = pendulum.now(tz="UTC")
-
         for item in _paginate(api_key, TEAMS_QUERY, "teams"):
             if pendulum.parse(item["updatedAt"]) >= current_start_date:
                 if pendulum.parse(item["updatedAt"]) <= current_end_date:
-                    yield _normalize_team(item)
+                    yield normalize_dictionaries(item)
 
     @dlt.resource(name="users", primary_key="id", write_disposition="merge")
     def users(
@@ -179,19 +188,28 @@ def linear_source(
             range_end="closed",
         ),
     ) -> Iterator[Dict[str, Any]]:
-        if updated_at.last_value:
-            current_start_date = pendulum.parse(updated_at.last_value)
-        else:
-            current_start_date = pendulum.parse(start_date)
-
-        if updated_at.end_value:
-            current_end_date = pendulum.parse(updated_at.end_value)
-        else:
-            current_end_date = pendulum.now(tz="UTC")
+        current_start_date, current_end_date = _get_date_range(updated_at, start_date)
 
         for item in _paginate(api_key, USERS_QUERY, "users"):
             if pendulum.parse(item["updatedAt"]) >= current_start_date:
                 if pendulum.parse(item["updatedAt"]) <= current_end_date:
-                    yield item
+                    yield normalize_dictionaries(item)
+
+    @dlt.resource(name="workflow_states", primary_key="id", write_disposition="merge")
+    def workflow_states(
+        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "updatedAt",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[Dict[str, Any]]:
+        current_start_date, current_end_date = _get_date_range(updated_at, start_date)
+
+        for item in _paginate(api_key, WORKFLOW_STATES_QUERY, "workflowStates"):
+            if pendulum.parse(item["updatedAt"]) >= current_start_date:
+                if pendulum.parse(item["updatedAt"]) <= current_end_date:
+                    yield normalize_dictionaries(item)
+    return [issues, projects, teams, users, workflow_states]
 
-    return issues, projects, teams, users

ingestr-0.13.80/ingestr/src/linear/helpers.py

@@ -0,0 +1,55 @@
+from typing import Any, Dict, Iterator, Optional
+
+import requests
+
+LINEAR_GRAPHQL_ENDPOINT = "https://api.linear.app/graphql"
+
+
+def _graphql(
+    api_key: str, query: str, variables: Optional[Dict[str, Any]] = None
+) -> Dict[str, Any]:
+    headers = {"Authorization": api_key, "Content-Type": "application/json"}
+    response = requests.post(
+        LINEAR_GRAPHQL_ENDPOINT,
+        json={"query": query, "variables": variables or {}},
+        headers=headers,
+    )
+    response.raise_for_status()
+    payload = response.json()
+    if "errors" in payload:
+        raise ValueError(str(payload["errors"]))
+    return payload["data"]
+
+
+def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
+    cursor: Optional[str] = None
+    while True:
+        data = _graphql(api_key, query, {"cursor": cursor})[root]
+        for item in data["nodes"]:
+            yield item
+        if not data["pageInfo"]["hasNextPage"]:
+            break
+        cursor = data["pageInfo"]["endCursor"]
+
+
+
+
+def normalize_dictionaries(item: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Automatically normalize dictionary fields by detecting their structure:
+    - Convert nested objects with 'id' field to {field_name}_id
+    - Convert objects with 'nodes' field to arrays
+    """
+    normalized_item = item.copy()
+    
+    for key, value in list(normalized_item.items()):
+        if isinstance(value, dict):
+            # If the dict has an 'id' field, replace with {key}_id
+            if 'id' in value:
+                normalized_item[f"{key}_id"] = value['id']
+                del normalized_item[key]
+            # If the dict has 'nodes' field, extract the nodes array
+            elif 'nodes' in value:
+                normalized_item[key] = value['nodes']
+    
+    return normalized_item

{ingestr-0.13.79 → ingestr-0.13.80}/ingestr/src/sources.py

@@ -702,6 +702,11 @@ class ShopifySource:
         return True
 
     def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Shopify takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
         source_fields = urlparse(uri)
         source_params = parse_qs(source_fields.query)
         api_key = source_params.get("api_key")
@@ -1003,6 +1008,16 @@ class FacebookAdsSource:
             facebook_insights_source,
         )
 
+        insights_max_wait_to_finish_seconds = source_params.get(
+            "insights_max_wait_to_finish_seconds", [60 * 60 * 4]
+        )
+        insights_max_wait_to_start_seconds = source_params.get(
+            "insights_max_wait_to_start_seconds", [60 * 30]
+        )
+        insights_max_async_sleep_seconds = source_params.get(
+            "insights_max_async_sleep_seconds", [20]
+        )
+
         endpoint = None
         if table in ["campaigns", "ad_sets", "ad_creatives", "ads", "leads"]:
             endpoint = table
@@ -1012,6 +1027,13 @@ class FacebookAdsSource:
                 account_id=account_id[0],
                 start_date=kwargs.get("interval_start"),
                 end_date=kwargs.get("interval_end"),
+                insights_max_wait_to_finish_seconds=insights_max_wait_to_finish_seconds[
+                    0
+                ],
+                insights_max_wait_to_start_seconds=insights_max_wait_to_start_seconds[
+                    0
+                ],
+                insights_max_async_sleep_seconds=insights_max_async_sleep_seconds[0],
             ).with_resources("facebook_insights")
         elif table.startswith("facebook_insights:"):
             # Parse custom breakdowns and metrics from table name
@@ -1652,6 +1674,11 @@ class TikTokSource:
         return True
 
     def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "TikTok takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
         endpoint = "custom_reports"
 
         parsed_uri = urlparse(uri)
@@ -1874,6 +1901,11 @@ class GoogleAnalyticsSource:
     def dlt_source(self, uri: str, table: str, **kwargs):
         import ingestr.src.google_analytics.helpers as helpers
 
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Google Analytics takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
         result = helpers.parse_google_analytics_uri(uri)
         credentials = result["credentials"]
         property_id = result["property_id"]
@@ -2246,6 +2278,11 @@ class LinkedInAdsSource:
         return True
 
     def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "LinkedIn Ads takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
         parsed_uri = urlparse(uri)
         source_fields = parse_qs(parsed_uri.query)
 
@@ -2329,6 +2366,11 @@ class ClickupSource:
         return True
 
     def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "ClickUp takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
         parsed_uri = urlparse(uri)
         params = parse_qs(parsed_uri.query)
         api_token = params.get("api_token")
@@ -2413,6 +2455,11 @@ class ApplovinMaxSource:
         return True
 
     def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "AppLovin Max takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
         parsed_uri = urlparse(uri)
         params = parse_qs(parsed_uri.query)
 
@@ -2505,6 +2552,11 @@ class PersonioSource:
 
     # applovin://?client_id=123&client_secret=123
     def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Personio takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
         parsed_uri = urlparse(uri)
         params = parse_qs(parsed_uri.query)
 
@@ -2595,6 +2647,11 @@ class PipedriveSource:
         return True
 
     def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Pipedrive takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
         parsed_uri = urlparse(uri)
         params = parse_qs(parsed_uri.query)
         api_key = params.get("api_token")
@@ -2677,6 +2734,11 @@ class FreshdeskSource:
         return True
 
     def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Freshdesk takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
         parsed_uri = urlparse(uri)
         domain = parsed_uri.netloc
         query = parsed_uri.query
@@ -2730,6 +2792,11 @@ class TrustpilotSource:
         return True
 
     def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Trustpilot takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
         parsed_uri = urlparse(uri)
         business_unit_id = parsed_uri.netloc
         params = parse_qs(parsed_uri.query)
@@ -2770,6 +2837,11 @@ class PhantombusterSource:
         return True
 
     def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Phantombuster takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
         # phantombuster://?api_key=<api_key>
         # source table = phantom_results:agent_id
         parsed_uri = urlparse(uri)
@@ -2923,6 +2995,11 @@ class SolidgateSource:
         return True
 
     def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Solidgate takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
         parsed_uri = urlparse(uri)
         query_params = parse_qs(parsed_uri.query)
         public_key = query_params.get("public_key")
@@ -3016,6 +3093,11 @@ class QuickBooksSource:
 
     # quickbooks://?company_id=<company_id>&client_id=<client_id>&client_secret=<client_secret>&refresh_token=<refresh>&access_token=<access_token>&environment=<env>&minor_version=<version>
     def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "QuickBooks takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
         parsed_uri = urlparse(uri)
 
         params = parse_qs(parsed_uri.query)
@@ -3085,6 +3167,11 @@ class IsocPulseSource:
         return True
 
     def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Internet Society Pulse takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
         parsed_uri = urlparse(uri)
         params = parse_qs(parsed_uri.query)
         token = params.get("token")
@@ -3120,6 +3207,11 @@ class PinterestSource:
         return True
 
     def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Pinterest takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
         parsed = urlparse(uri)
         params = parse_qs(parsed.query)
         access_token = params.get("access_token")
@@ -3154,13 +3246,18 @@ class LinearSource:
         return True
 
     def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Linear takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
         parsed_uri = urlparse(uri)
         params = parse_qs(parsed_uri.query)
         api_key = params.get("api_key")
         if api_key is None:
             raise MissingValueError("api_key", "Linear")
 
-        if table not in ["issues", "projects", "teams", "users"]:
+        if table not in ["issues", "projects", "teams", "users", "workflow_states"]:
             raise UnsupportedResourceError(table, "Linear")
 
         start_date = kwargs.get("interval_start")
@@ -3187,6 +3284,11 @@ class ZoomSource:
         return True
 
     def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Zoom takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
         parsed = urlparse(uri)
         params = parse_qs(parsed.query)
         client_id = params.get("client_id")
@@ -3228,6 +3330,11 @@ class InfluxDBSource:
         return True
 
     def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "InfluxDB takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
         parsed_uri = urlparse(uri)
         params = parse_qs(parsed_uri.query)
         host = parsed_uri.hostname

ingestr-0.13.79/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.79"

ingestr-0.13.79/ingestr/src/linear/helpers.py

@@ -1,72 +0,0 @@
-from typing import Any, Dict, Iterator, Optional
-
-import requests
-
-LINEAR_GRAPHQL_ENDPOINT = "https://api.linear.app/graphql"
-
-
-def _graphql(
-    api_key: str, query: str, variables: Optional[Dict[str, Any]] = None
-) -> Dict[str, Any]:
-    headers = {"Authorization": api_key, "Content-Type": "application/json"}
-    response = requests.post(
-        LINEAR_GRAPHQL_ENDPOINT,
-        json={"query": query, "variables": variables or {}},
-        headers=headers,
-    )
-    response.raise_for_status()
-    payload = response.json()
-    if "errors" in payload:
-        raise ValueError(str(payload["errors"]))
-    return payload["data"]
-
-
-def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
-    cursor: Optional[str] = None
-    while True:
-        data = _graphql(api_key, query, {"cursor": cursor})[root]
-        for item in data["nodes"]:
-            yield item
-        if not data["pageInfo"]["hasNextPage"]:
-            break
-        cursor = data["pageInfo"]["endCursor"]
-
-
-def _normalize_issue(item: Dict[str, Any]) -> Dict[str, Any]:
-    field_mapping = {
-        "assignee": "assignee_id",
-        "creator": "creator_id",
-        "state": "state_id",
-        "cycle": "cycle_id",
-        "project": "project_id",
-    }
-    for key, value in field_mapping.items():
-        if item.get(key):
-            item[value] = item[key]["id"]
-            del item[key]
-        else:
-            item[value] = None
-            del item[key]
-    json_fields = [
-        "comments",
-        "subscribers",
-        "attachments",
-        "labels",
-        "subtasks",
-        "projects",
-        "memberships",
-        "members",
-    ]
-    for field in json_fields:
-        if item.get(field):
-            item[f"{field}"] = item[field].get("nodes", [])
-
-    return item
-
-
-def _normalize_team(item: Dict[str, Any]) -> Dict[str, Any]:
-    json_fields = ["memberships", "members", "projects"]
-    for field in json_fields:
-        if item.get(field):
-            item[f"{field}"] = item[field].get("nodes", [])
-    return item