ingestr 0.13.61__py3-none-any.whl → 0.13.62__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.61"
+version = "v0.13.62"

ingestr/src/factory.py

@@ -48,6 +48,7 @@ from ingestr.src.sources import (
     KafkaSource,
     KinesisSource,
     KlaviyoSource,
+    LinearSource,
     LinkedInAdsSource,
     LocalCsvSource,
     MixpanelSource,
@@ -70,6 +71,7 @@ from ingestr.src.sources import (
     TikTokSource,
     TrustpilotSource,
     ZendeskSource,
+    ZoomSource,
 )
 
 SQL_SOURCE_SCHEMES = [
@@ -162,6 +164,7 @@ class SourceDestinationFactory:
         "appstore": AppleAppStoreSource,
         "gs": GCSSource,
         "linkedinads": LinkedInAdsSource,
+        "linear": LinearSource,
         "applovin": AppLovinSource,
         "applovinmax": ApplovinMaxSource,
         "salesforce": SalesforceSource,
@@ -180,6 +183,7 @@ class SourceDestinationFactory:
         "smartsheet": SmartsheetSource,
         "sftp": SFTPSource,
         "pinterest": PinterestSource,
+        "zoom": ZoomSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr/src/linear/__init__.py

@@ -0,0 +1,183 @@
+from typing import Any, Dict, Iterable, Iterator, Optional
+
+import dlt
+import pendulum
+import requests
+
+LINEAR_GRAPHQL_ENDPOINT = "https://api.linear.app/graphql"
+
+
+def _graphql(
+    api_key: str, query: str, variables: Optional[Dict[str, Any]] = None
+) -> Dict[str, Any]:
+    headers = {"Authorization": api_key, "Content-Type": "application/json"}
+    response = requests.post(
+        LINEAR_GRAPHQL_ENDPOINT,
+        json={"query": query, "variables": variables or {}},
+        headers=headers,
+    )
+    response.raise_for_status()
+    payload = response.json()
+    if "errors" in payload:
+        raise ValueError(str(payload["errors"]))
+    return payload["data"]
+
+
+def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
+    cursor: Optional[str] = None
+    while True:
+        data = _graphql(api_key, query, {"cursor": cursor})[root]
+        for item in data["nodes"]:
+            yield item
+        if not data["pageInfo"]["hasNextPage"]:
+            break
+        cursor = data["pageInfo"]["endCursor"]
+
+
+ISSUES_QUERY = """
+query Issues($cursor: String) {
+  issues(first: 50, after: $cursor) {
+    nodes {
+      id
+      title
+      description
+      createdAt
+      updatedAt
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+PROJECTS_QUERY = """
+query Projects($cursor: String) {
+  projects(first: 50, after: $cursor) {
+    nodes {
+      id
+      name
+      description
+      createdAt
+      updatedAt
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+TEAMS_QUERY = """
+query Teams($cursor: String) {
+  teams(first: 50, after: $cursor) {
+    nodes {
+      id
+      name
+      key
+      description
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+USERS_QUERY = """
+query Users($cursor: String) {
+  users(first: 50, after: $cursor) {
+    nodes {
+      id
+      name
+      displayName
+      email
+      createdAt
+      updatedAt
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+
+@dlt.source(name="linear", max_table_nesting=0)
+def linear_source(
+    api_key: str,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None = None,
+) -> Iterable[dlt.sources.DltResource]:
+    @dlt.resource(name="issues", primary_key="id", write_disposition="merge")
+    def issues(
+        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "updatedAt",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[Dict[str, Any]]:
+        if updated_at.last_value:
+            current_start_date = pendulum.parse(updated_at.last_value)
+        else:
+            current_start_date = pendulum.parse(start_date)
+
+        if updated_at.end_value:
+            current_end_date = pendulum.parse(updated_at.end_value)
+        else:
+            current_end_date = pendulum.now(tz="UTC")
+
+        for item in _paginate(api_key, ISSUES_QUERY, "issues"):
+            if pendulum.parse(item["updatedAt"]) >= current_start_date:
+                if pendulum.parse(item["updatedAt"]) <= current_end_date:
+                    yield item
+
+    @dlt.resource(name="projects", primary_key="id", write_disposition="merge")
+    def projects(
+        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "updatedAt",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[Dict[str, Any]]:
+        if updated_at.last_value:
+            current_start_date = pendulum.parse(updated_at.last_value)
+        else:
+            current_start_date = pendulum.parse(start_date)
+
+        if updated_at.end_value:
+            current_end_date = pendulum.parse(updated_at.end_value)
+        else:
+            current_end_date = pendulum.now(tz="UTC")
+
+        for item in _paginate(api_key, PROJECTS_QUERY, "projects"):
+            if pendulum.parse(item["updatedAt"]) >= current_start_date:
+                if pendulum.parse(item["updatedAt"]) <= current_end_date:
+                    yield item
+
+    @dlt.resource(name="teams", primary_key="id", write_disposition="merge")
+    def teams() -> Iterator[Dict[str, Any]]:
+        yield from _paginate(api_key, TEAMS_QUERY, "teams")
+
+    @dlt.resource(name="users", primary_key="id", write_disposition="merge")
+    def users(
+        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "updatedAt",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[Dict[str, Any]]:
+        if updated_at.last_value:
+            current_start_date = pendulum.parse(updated_at.last_value)
+        else:
+            current_start_date = pendulum.parse(start_date)
+
+        if updated_at.end_value:
+            current_end_date = pendulum.parse(updated_at.end_value)
+        else:
+            current_end_date = pendulum.now(tz="UTC")
+
+        for item in _paginate(api_key, USERS_QUERY, "users"):
+            if pendulum.parse(item["updatedAt"]) >= current_start_date:
+                if pendulum.parse(item["updatedAt"]) <= current_end_date:
+                    yield item
+
+    return issues, projects, teams, users

ingestr/src/sources.py

@@ -2851,3 +2851,77 @@ class PinterestSource:
             start_date=start_date,
             end_date=end_date,
         ).with_resources(table)
+
+
+class LinearSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+        api_key = params.get("api_key")
+        if api_key is None:
+            raise MissingValueError("api_key", "Linear")
+
+        if table not in ["issues", "projects", "teams", "users"]:
+            raise UnsupportedResourceError(table, "Linear")
+
+        start_date = kwargs.get("interval_start")
+        if start_date is not None:
+            start_date = ensure_pendulum_datetime(start_date)
+        else:
+            start_date = pendulum.datetime(2020, 1, 1).in_tz("UTC")
+
+        end_date = kwargs.get("interval_end")
+        if end_date is not None:
+            end_date = end_date = ensure_pendulum_datetime(end_date).in_tz("UTC")
+
+        from ingestr.src.linear import linear_source
+
+        return linear_source(
+            api_key=api_key[0],
+            start_date=start_date,
+            end_date=end_date,
+        ).with_resources(table)
+
+
+class ZoomSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed = urlparse(uri)
+        params = parse_qs(parsed.query)
+        client_id = params.get("client_id")
+        client_secret = params.get("client_secret")
+        account_id = params.get("account_id")
+
+        if not (client_id and client_secret and account_id):
+            raise MissingValueError(
+                "client_id/client_secret/account_id",
+                "Zoom",
+            )
+
+        start_date = kwargs.get("interval_start")
+        if start_date is not None:
+            start_date = ensure_pendulum_datetime(start_date)
+        else:
+            start_date = pendulum.datetime(2020, 1, 26).in_tz("UTC")
+
+        end_date = kwargs.get("interval_end")
+        if end_date is not None:
+            end_date = end_date = ensure_pendulum_datetime(end_date).in_tz("UTC")
+
+        from ingestr.src.zoom import zoom_source
+
+        if table not in {"meetings"}:
+            raise UnsupportedResourceError(table, "Zoom")
+
+        return zoom_source(
+            client_id=client_id[0],
+            client_secret=client_secret[0],
+            account_id=account_id[0],
+            start_date=start_date,
+            end_date=end_date,
+        ).with_resources(table)

ingestr/src/zoom/__init__.py

@@ -0,0 +1,55 @@
+from typing import Any, Dict, Iterable, Sequence
+
+import dlt
+import pendulum
+from dlt.common.typing import TAnyDateTime, TDataItem
+from dlt.sources import DltResource
+
+from .helpers import ZoomClient
+
+
+@dlt.source(name="zoom", max_table_nesting=0)
+def zoom_source(
+    client_id: str,
+    client_secret: str,
+    account_id: str,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None = None,
+) -> Sequence[DltResource]:
+    """Create a Zoom source with meetings resource for all users in the account."""
+    client = ZoomClient(
+        client_id=client_id,
+        client_secret=client_secret,
+        account_id=account_id,
+    )
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def meetings(
+        datetime: dlt.sources.incremental[TAnyDateTime] = dlt.sources.incremental(
+            "start_time",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date is not None else None,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        if datetime.last_value:
+            start_dt = pendulum.parse(datetime.last_value)
+        else:
+            start_dt = pendulum.parse(start_date)
+
+        if end_date is None:
+            end_dt = pendulum.now("UTC")
+        else:
+            end_dt = pendulum.parse(datetime.end_value)
+        base_params: Dict[str, Any] = {
+            "type": "scheduled",
+            "page_size": 300,
+            "from": start_dt.to_date_string(),
+            "to": end_dt.to_date_string(),
+        }
+        for user in client.get_users():
+            user_id = user["id"]
+            yield from client.get_meetings(user_id, base_params)
+
+    return meetings

ingestr/src/zoom/helpers.py

@@ -0,0 +1,76 @@
+import time
+from typing import Any, Dict, Iterator, Optional
+
+from ingestr.src.http_client import create_client
+
+
+class ZoomClient:
+    """Minimal Zoom API client supporting Server-to-Server OAuth."""
+
+    def __init__(
+        self,
+        client_id: Optional[str] = None,
+        client_secret: Optional[str] = None,
+        account_id: Optional[str] = None,
+    ) -> None:
+        self.client_id = client_id
+        self.client_secret = client_secret
+        self.account_id = account_id
+        self.token_expires_at: float = 0
+        self.base_url = "https://api.zoom.us/v2"
+        self.session = create_client()
+        self._refresh_access_token()
+
+    def _refresh_access_token(self) -> None:
+        token_url = "https://zoom.us/oauth/token"
+        auth = (self.client_id, self.client_secret)
+        resp = self.session.post(
+            token_url,
+            params={"grant_type": "account_credentials", "account_id": self.account_id},
+            auth=auth,
+        )
+        resp.raise_for_status()
+        data = resp.json()
+        self.access_token = data.get("access_token")
+        self.token_expires_at = time.time() + data.get("expires_in", 3600)
+
+    def _ensure_token(self) -> None:
+        if self.access_token is None or self.token_expires_at <= time.time():
+            self._refresh_access_token()
+
+    def _headers(self) -> Dict[str, str]:
+        self._ensure_token()
+        return {
+            "Authorization": f"Bearer {self.access_token}",
+            "Accept": "application/json",
+        }
+
+    def get_users(self) -> Iterator[Dict[str, Any]]:
+        url = f"{self.base_url}/users"
+        params = {"page_size": 1000}
+        while True:
+            response = self.session.get(url, headers=self._headers(), params=params)
+            response.raise_for_status()
+            data = response.json()
+            for user in data.get("users", []):
+                yield user
+            token = data.get("next_page_token")
+            if not token:
+                break
+            params["next_page_token"] = token
+
+    def get_meetings(
+        self, user_id: str, params: Dict[str, Any]
+    ) -> Iterator[Dict[str, Any]]:
+        url = f"{self.base_url}/users/{user_id}/meetings"
+        while True:
+            response = self.session.get(url, headers=self._headers(), params=params)
+            response.raise_for_status()
+            data = response.json()
+            for item in data.get("meetings", []):
+                item["zoom_user_id"] = user_id
+                yield item
+            token = data.get("next_page_token")
+            if not token:
+                break
+            params["next_page_token"] = token

{ingestr-0.13.61.dist-info → ingestr-0.13.62.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.61
+Version: 0.13.62
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.61.dist-info → ingestr-0.13.62.dist-info}/RECORD

@@ -2,16 +2,16 @@ ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
 ingestr/main.py,sha256=taDyHyaVSpB17iNLl8zA0gmr4CqDO-MSTQX1CaRBB9U,26364
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
-ingestr/src/buildinfo.py,sha256=slTtbF5rCzvz-7Z_L3XF0kUk-fOkQu3INJIPtywQb5Y,21
+ingestr/src/buildinfo.py,sha256=TQVet9YuexpAbyia8_nOwytK_io6rEVmB-flmAr4z8E,21
 ingestr/src/destinations.py,sha256=YU7c5cNqViCIJ9NDhgGuorh5jMvLi7yViPWpEJ57Xx0,18788
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=j_ANB1X5BUa_iUOLqreHMGWnDPFV4bGgd0lL35lZ6XE,6432
+ingestr/src/factory.py,sha256=nYXINj54QtMGuQv9oCXbHO7DFuAudqZfijkSvSXEVj8,6526
 ingestr/src/filters.py,sha256=LLecXe9QkLFkFLUZ92OXNdcANr1a8edDxrflc2ko_KA,1452
 ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,732
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=SZo44oi7DBTEJrAvVhbj0lpXWd4aGSQTMgwZ17lgO9c,99048
+ingestr/src/sources.py,sha256=nTzCi_RZhUesBkXyUTAIoAUgj5iZSQJ_D8mId8rq8mE,101467
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -81,6 +81,7 @@ ingestr/src/kinesis/helpers.py,sha256=SO2cFmWNGcykUYmjHdfxWsOQSkLQXyhFtfWnkcUOM0
 ingestr/src/klaviyo/__init__.py,sha256=o_noUgbxLk36s4f9W56_ibPorF0n7kVapPUlV0p-jfA,7875
 ingestr/src/klaviyo/client.py,sha256=tPj79ia7AW0ZOJhzlKNPCliGbdojRNwUFp8HvB2ym5s,7434
 ingestr/src/klaviyo/helpers.py,sha256=_i-SHffhv25feLDcjy6Blj1UxYLISCwVCMgGtrlnYHk,496
+ingestr/src/linear/__init__.py,sha256=ITMLsuLjrGYx3bTsEK1cdPUkowJYCdAII_ucci_lGDQ,5422
 ingestr/src/linkedin_ads/__init__.py,sha256=CAPWFyV24loziiphbLmODxZUXZJwm4JxlFkr56q0jfo,1855
 ingestr/src/linkedin_ads/dimension_time_enum.py,sha256=EmHRdkFyTAfo4chGjThrwqffWJxmAadZMbpTvf0xkQc,198
 ingestr/src/linkedin_ads/helpers.py,sha256=eUWudRVlXl4kqIhfXQ1eVsUpZwJn7UFqKSpnbLfxzds,4498
@@ -134,6 +135,8 @@ ingestr/src/zendesk/helpers/__init__.py,sha256=YTJejCiUjfIcsj9FrkY0l-JGYDI7RRte1
 ingestr/src/zendesk/helpers/api_helpers.py,sha256=dMkNn4ZQXgJTDOXAAXdmRt41phNFoRhYyPaLJih0pZY,4184
 ingestr/src/zendesk/helpers/credentials.py,sha256=EWyi0ZlxWFgd1huD86KNF4dApLHgmabqWksFpEg1cf0,1332
 ingestr/src/zendesk/helpers/talk_api.py,sha256=TSVSOErsBZvxcX91LMhAgvy6yLSYvpuVfOyKViOHtvA,4718
+ingestr/src/zoom/__init__.py,sha256=6NdHXLv438FLD-cUEgmgnuJi__70-C88bTGOLqYHqbQ,1736
+ingestr/src/zoom/helpers.py,sha256=Y6fjIpQTWUAkKXJKfwjJnZc6wlFlqzJfWqq34WZhrcU,2669
 ingestr/testdata/.gitignore,sha256=DFzYYOpqdTiT7S1HjCT-jffZSmEvFZge295_upAB0FY,13
 ingestr/testdata/create_replace.csv,sha256=TQDbOSkRKq9ZZv1d68Qjwh94aIyUQ-oEwxpJIrd3YK8,1060
 ingestr/testdata/delete_insert_expected.csv,sha256=wbj7uboVWwm3sNMh1n7f4-OKFEQJv1s96snjEHp9nkg,336
@@ -143,8 +146,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
-ingestr-0.13.61.dist-info/METADATA,sha256=0VM6DuyeROIX3TUb8FBXRMiUg4KktwK8Gr95ZespXHE,15027
-ingestr-0.13.61.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.61.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.61.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.61.dist-info/RECORD,,
+ingestr-0.13.62.dist-info/METADATA,sha256=GdxhHlw_v3U_5vL1ZTbDFxRMr7RLBnWDA-dVwG74OtQ,15027
+ingestr-0.13.62.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.62.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.62.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.62.dist-info/RECORD,,