ingestr 0.13.64__py3-none-any.whl → 0.13.65__py3-none-any.whl

ingestr/src/attio/__init__.py

@@ -20,13 +20,15 @@ def attio_source(
             "created_at": {"data_type": "timestamp", "partition": True},
         },
     )
+    # https://docs.attio.com/rest-api/endpoint-reference/objects/list-objects - does not support pagination
     def fetch_objects() -> Iterator[dict]:
         if len(params) != 0:
             raise ValueError("Objects table must be in the format `objects`")
 
         path = "objects"
-        yield attio_client.fetch_data(path, "get")
+        yield attio_client.fetch_all(path, "get")
 
+    # https://docs.attio.com/rest-api/endpoint-reference/records/list-records
     @dlt.resource(
         name="records",
         write_disposition="replace",
@@ -39,12 +41,12 @@ def attio_source(
             raise ValueError(
                 "Records table must be in the format `records:{object_api_slug}`"
             )
-
         object_id = params[0]
         path = f"objects/{object_id}/records/query"
 
-        yield attio_client.fetch_data(path, "post")
+        yield attio_client.fetch_paginated(path, "post")
 
+    # https://docs.attio.com/rest-api/endpoint-reference/lists/list-all-lists -- does not support pagination
     @dlt.resource(
         name="lists",
         write_disposition="replace",
@@ -54,8 +56,9 @@ def attio_source(
     )
     def fetch_lists() -> Iterator[dict]:
         path = "lists"
-        yield attio_client.fetch_data(path, "get")
+        yield attio_client.fetch_all(path, "get")
 
+    # https://docs.attio.com/rest-api/endpoint-reference/entries/list-entries
     @dlt.resource(
         name="list_entries",
         write_disposition="replace",
@@ -70,7 +73,7 @@ def attio_source(
             )
         path = f"lists/{params[0]}/entries/query"
 
-        yield attio_client.fetch_data(path, "post")
+        yield attio_client.fetch_paginated(path, "post")
 
     @dlt.resource(
         name="all_list_entries",
@@ -85,10 +88,10 @@ def attio_source(
                 "All list entries table must be in the format `all_list_entries:{object_api_slug}`"
             )
         path = "lists"
-        for lst in attio_client.fetch_data(path, "get"):
+        for lst in attio_client.fetch_all(path, "get"):
             if params[0] in lst["parent_object"]:
                 path = f"lists/{lst['id']['list_id']}/entries/query"
-                yield from attio_client.fetch_data(path, "post")
+                yield from attio_client.fetch_paginated(path, "post")
 
     return (
         fetch_objects,

ingestr/src/attio/helpers.py

@@ -10,42 +10,53 @@ class AttioClient:
         }
         self.client = create_client()
 
-    def fetch_data(self, path: str, method: str, limit: int = 1000, params=None):
+    def fetch_paginated(self, path: str, method: str, limit: int = 1000, params=None):
         url = f"{self.base_url}/{path}"
         if params is None:
             params = {}
         offset = 0
         while True:
-            query_params = {**params, "limit": limit, "offset": offset}
+            query_params = {"limit": limit, "offset": offset, **params}
             if method == "get":
                 response = self.client.get(
                     url, headers=self.headers, params=query_params
                 )
             else:
-                response = self.client.post(
-                    url, headers=self.headers, params=query_params
-                )
+                json_body = {**params, "limit": limit, "offset": offset}
+                response = self.client.post(url, headers=self.headers, json=json_body)
 
             if response.status_code != 200:
                 raise Exception(f"HTTP {response.status_code} error: {response.text}")
 
             response_data = response.json()
             if "data" not in response_data:
-                print(f"API Response: {response_data}")
                 raise Exception(
                     "Attio API returned a response without the expected data"
                 )
 
             data = response_data["data"]
-
             for item in data:
                 flat_item = flatten_item(item)
                 yield flat_item
-
             if len(data) < limit:
                 break
+
             offset += limit
 
+    def fetch_all(self, path: str, method: str = "get", params=None):
+        url = f"{self.base_url}/{path}"
+        params = params or {}
+
+        if method == "get":
+            response = self.client.get(url, headers=self.headers, params=params)
+        else:
+            response = self.client.post(url, headers=self.headers, json=params)
+
+        response.raise_for_status()
+        data = response.json().get("data", [])
+        for item in data:
+            yield flatten_item(item)
+
 
 def flatten_item(item: dict) -> dict:
     if "id" in item:

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.64"
+version = "v0.13.65"

ingestr/src/linear/__init__.py

@@ -1,38 +1,9 @@
-from typing import Any, Dict, Iterable, Iterator, Optional
+from typing import Any, Dict, Iterable, Iterator
 
 import dlt
 import pendulum
-import requests
-
-LINEAR_GRAPHQL_ENDPOINT = "https://api.linear.app/graphql"
-
-
-def _graphql(
-    api_key: str, query: str, variables: Optional[Dict[str, Any]] = None
-) -> Dict[str, Any]:
-    headers = {"Authorization": api_key, "Content-Type": "application/json"}
-    response = requests.post(
-        LINEAR_GRAPHQL_ENDPOINT,
-        json={"query": query, "variables": variables or {}},
-        headers=headers,
-    )
-    response.raise_for_status()
-    payload = response.json()
-    if "errors" in payload:
-        raise ValueError(str(payload["errors"]))
-    return payload["data"]
-
-
-def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
-    cursor: Optional[str] = None
-    while True:
-        data = _graphql(api_key, query, {"cursor": cursor})[root]
-        for item in data["nodes"]:
-            yield item
-        if not data["pageInfo"]["hasNextPage"]:
-            break
-        cursor = data["pageInfo"]["endCursor"]
 
+from .helpers import  _paginate, _normalize_issue, _normalize_team
 
 ISSUES_QUERY = """
 query Issues($cursor: String) {
@@ -43,6 +14,17 @@ query Issues($cursor: String) {
       description
       createdAt
       updatedAt
+      creator { id }
+      assignee { id}
+      state { id}
+      labels { nodes { id } }
+      cycle { id}
+      project { id }
+      subtasks: children { nodes { id title } }
+      comments(first: 250) { nodes { id body } }
+      priority
+      attachments { nodes { id } }
+      subscribers { nodes { id } }
     }
     pageInfo { hasNextPage endCursor }
   }
@@ -58,6 +40,10 @@ query Projects($cursor: String) {
       description
       createdAt
       updatedAt
+      health
+      priority
+      targetDate
+      lead { id }
     }
     pageInfo { hasNextPage endCursor }
   }
@@ -72,6 +58,11 @@ query Teams($cursor: String) {
       name
       key
       description
+      updatedAt
+      createdAt
+      memberships { nodes { id } }
+      members { nodes { id } }
+      projects { nodes { id } }
     }
     pageInfo { hasNextPage endCursor }
   }
@@ -124,7 +115,7 @@ def linear_source(
         for item in _paginate(api_key, ISSUES_QUERY, "issues"):
             if pendulum.parse(item["updatedAt"]) >= current_start_date:
                 if pendulum.parse(item["updatedAt"]) <= current_end_date:
-                    yield item
+                    yield _normalize_issue(item)
 
     @dlt.resource(name="projects", primary_key="id", write_disposition="merge")
     def projects(
@@ -152,8 +143,29 @@ def linear_source(
                     yield item
 
     @dlt.resource(name="teams", primary_key="id", write_disposition="merge")
-    def teams() -> Iterator[Dict[str, Any]]:
-        yield from _paginate(api_key, TEAMS_QUERY, "teams")
+    def teams( updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "updatedAt",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
+            range_start="closed",
+            range_end="closed",
+        ),) -> Iterator[Dict[str, Any]]:
+        print(start_date)
+        if updated_at.last_value:
+            current_start_date = pendulum.parse(updated_at.last_value)
+        else:
+            current_start_date = pendulum.parse(start_date)
+        print(current_start_date)
+
+        if updated_at.end_value:
+            current_end_date = pendulum.parse(updated_at.end_value)
+        else:
+            current_end_date = pendulum.now(tz="UTC")
+
+        for item in _paginate(api_key, TEAMS_QUERY, "teams"):
+            if pendulum.parse(item["updatedAt"]) >= current_start_date:
+                if pendulum.parse(item["updatedAt"]) <= current_end_date:
+                    yield _normalize_team(item)
 
     @dlt.resource(name="users", primary_key="id", write_disposition="merge")
     def users(

ingestr/src/linear/helpers.py

@@ -0,0 +1,60 @@
+import json
+from typing import Any, Dict, Iterator, Optional
+
+import requests
+
+LINEAR_GRAPHQL_ENDPOINT = "https://api.linear.app/graphql"
+
+def _graphql(
+    api_key: str, query: str, variables: Optional[Dict[str, Any]] = None
+) -> Dict[str, Any]:
+    headers = {"Authorization": api_key, "Content-Type": "application/json"}
+    response = requests.post(
+        LINEAR_GRAPHQL_ENDPOINT,
+        json={"query": query, "variables": variables or {}},
+        headers=headers,
+    )
+    response.raise_for_status()
+    payload = response.json()
+    if "errors" in payload:
+        raise ValueError(str(payload["errors"]))
+    return payload["data"]
+
+def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
+    cursor: Optional[str] = None
+    while True:
+        data = _graphql(api_key, query, {"cursor": cursor})[root]
+        for item in data["nodes"]:
+            yield item
+        if not data["pageInfo"]["hasNextPage"]:
+            break
+        cursor = data["pageInfo"]["endCursor"]
+
+def _normalize_issue(item: Dict[str, Any]) -> Dict[str, Any]:
+    field_mapping = {
+        "assignee": "assignee_id",
+        "creator": "creator_id",
+        "state": "state_id",
+        "cycle": "cycle_id",
+        "project": "project_id",
+    }
+    for key, value in field_mapping.items():
+        if item.get(key):
+            item[value] = item[key]["id"]
+            del item[key]
+        else:
+            item[value] = None
+            del item[key]
+    json_fields = ["comments", "subscribers", "attachments", "labels", "subtasks","projects", "memberships", "members"]
+    for field in json_fields:
+        if item.get(field):
+            item[f"{field}"] = item[field].get("nodes", [])
+                
+    return item
+
+def _normalize_team(item: Dict[str, Any]) -> Dict[str, Any]:
+    json_fields = ["memberships", "members", "projects"]
+    for field in json_fields:
+        if item.get(field):
+            item[f"{field}"] = item[field].get("nodes", [])
+    return item

{ingestr-0.13.64.dist-info → ingestr-0.13.65.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.64
+Version: 0.13.65
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.64.dist-info → ingestr-0.13.65.dist-info}/RECORD

@@ -2,7 +2,7 @@ ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
 ingestr/main.py,sha256=taDyHyaVSpB17iNLl8zA0gmr4CqDO-MSTQX1CaRBB9U,26364
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
-ingestr/src/buildinfo.py,sha256=PuIV-cer5xdwwdtvFJ-yA_CjyfoO-SckFPSRHGfPM4I,21
+ingestr/src/buildinfo.py,sha256=RDAMEy23q-LmXSYODsQMAghvn5syzLPD4mQO_GpxC0c,21
 ingestr/src/destinations.py,sha256=ZJTbTn1K9oXinL19dTGQDUrft5C9fjrpSlTw1CLQhuM,21749
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
 ingestr/src/factory.py,sha256=AJCvlK4M1sIpAAks1K-xsR_uxziIxru74mj572zixhg,6546
@@ -31,8 +31,8 @@ ingestr/src/arrow/__init__.py,sha256=8fEntgHseKjFMiPQIzxYzw_raicNsEgnveLi1IzBca0
 ingestr/src/asana_source/__init__.py,sha256=QwQTCb5PXts8I4wLHG9UfRP-5ChfjSe88XAVfxMV5Ag,8183
 ingestr/src/asana_source/helpers.py,sha256=PukcdDQWIGqnGxuuobbLw4hUy4-t6gxXg_XywR7Lg9M,375
 ingestr/src/asana_source/settings.py,sha256=-2tpdkwh04RvLKFvwQodnFLYn9MaxOO1hsebGnDQMTU,2829
-ingestr/src/attio/__init__.py,sha256=D21EK02HQxDtHoJHVHtM01sU4ZSK26WzFjLqqpVDdK0,2859
-ingestr/src/attio/helpers.py,sha256=QvB-0BV_Z-cvMTBZDwOCuhxY1cB5PraPdrDkNyQ5TsM,1715
+ingestr/src/attio/__init__.py,sha256=CLejJjp5vGkt6r18nfNNZ-Xjc1SZgQ5IlcBW5XFQR90,3243
+ingestr/src/attio/helpers.py,sha256=fCySmG5E6Iyh3Nm9a-HGbHNedxPH_2_otXYMTQsCibw,2185
 ingestr/src/chess/__init__.py,sha256=y0Q8aKBigeKf3N7wuB_gadMQjVJzBPUT8Jhp1ObEWjk,6812
 ingestr/src/chess/helpers.py,sha256=v1HTImOMjAF7AzZUPDIuHu00e7ut0o5y1kWcVYo4QZw,549
 ingestr/src/chess/settings.py,sha256=p0RlCGgtXUacPDEvZmwzSWmzX0Apj1riwfz-nrMK89k,158
@@ -81,7 +81,8 @@ ingestr/src/kinesis/helpers.py,sha256=SO2cFmWNGcykUYmjHdfxWsOQSkLQXyhFtfWnkcUOM0
 ingestr/src/klaviyo/__init__.py,sha256=o_noUgbxLk36s4f9W56_ibPorF0n7kVapPUlV0p-jfA,7875
 ingestr/src/klaviyo/client.py,sha256=tPj79ia7AW0ZOJhzlKNPCliGbdojRNwUFp8HvB2ym5s,7434
 ingestr/src/klaviyo/helpers.py,sha256=_i-SHffhv25feLDcjy6Blj1UxYLISCwVCMgGtrlnYHk,496
-ingestr/src/linear/__init__.py,sha256=ITMLsuLjrGYx3bTsEK1cdPUkowJYCdAII_ucci_lGDQ,5422
+ingestr/src/linear/__init__.py,sha256=attlRyodShvAZ5dmDJXgoKrYhwElpLMQTSaRaAGEqC0,5941
+ingestr/src/linear/helpers.py,sha256=VR_CBgTfMVTH6ULcSLKyrssGoJpJx8VFZrmBeYZzFfc,1995
 ingestr/src/linkedin_ads/__init__.py,sha256=CAPWFyV24loziiphbLmODxZUXZJwm4JxlFkr56q0jfo,1855
 ingestr/src/linkedin_ads/dimension_time_enum.py,sha256=EmHRdkFyTAfo4chGjThrwqffWJxmAadZMbpTvf0xkQc,198
 ingestr/src/linkedin_ads/helpers.py,sha256=eUWudRVlXl4kqIhfXQ1eVsUpZwJn7UFqKSpnbLfxzds,4498
@@ -146,8 +147,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
-ingestr-0.13.64.dist-info/METADATA,sha256=QaBM1vQbQPYKgzDga2AY6uKibQbd1SkavE3dw92Pw-o,15027
-ingestr-0.13.64.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.64.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.64.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.64.dist-info/RECORD,,
+ingestr-0.13.65.dist-info/METADATA,sha256=PWjju7xvb3O9Ya0IRwj-zti34_sN6sGSY3YbROP3KKs,15027
+ingestr-0.13.65.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.65.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.65.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.65.dist-info/RECORD,,