ingestr 0.13.61__tar.gz → 0.13.63__tar.gz

{ingestr-0.13.61 → ingestr-0.13.63}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.61
+Version: 0.13.63
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.61 → ingestr-0.13.63}/docs/.vitepress/config.mjs

@@ -71,6 +71,7 @@ export default defineConfig({
               { text: "AWS Athena", link: "/supported-sources/athena.md" },
               { text: "AWS Redshift", link: "/supported-sources/redshift.md" },
               { text: "ClickHouse", link: "/supported-sources/clickhouse.md" },
+              { text: "CrateDB", link: "/supported-sources/cratedb.md" },
               { text: "Databricks", link: "/supported-sources/databricks.md" },
               { text: "DuckDB", link: "/supported-sources/duckdb.md" },
               { text: "DynamoDB", link: "/supported-sources/dynamodb.md" },
@@ -133,6 +134,7 @@ export default defineConfig({
               { text: "HubSpot", link: "/supported-sources/hubspot.md" },
               { text: "Internet Society Pulse", link: "/supported-sources/isoc-pulse.md" },
               { text: "Klaviyo", link: "/supported-sources/klaviyo.md" },
+              { text: "Linear", link: "/supported-sources/linear.md" },
               { text: "LinkedIn Ads", link: "/supported-sources/linkedin_ads.md" },
               { text: "Mixpanel", link: "/supported-sources/mixpanel.md" },
               { text: "Notion", link: "/supported-sources/notion.md" },
@@ -151,6 +153,7 @@ export default defineConfig({
               { text: "Stripe", link: "/supported-sources/stripe.md" },
               { text: "TikTok Ads", link: "/supported-sources/tiktok-ads.md" },
               { text: "Zendesk", link: "/supported-sources/zendesk.md" },
+              { text: "Zoom", link: "/supported-sources/zoom.md" },
             ],
           },
         ],

{ingestr-0.13.61 → ingestr-0.13.63}/docs/index.md

@@ -27,5 +27,5 @@ features:
 ---
 <div style="margin-top: 12px; line-height: 2em; text-align: center;">
 
-<Badge type="info" text="Postgres" /> <Badge type="danger" text="BigQuery" /> <Badge type="tip" text="Snowflake" /> <Badge type="warning" text="Redshift" /> <Badge type="info" text="Databricks" /> <Badge type="danger" text="DuckDB" /> <Badge type="tip" text="Microsoft SQL Server" /> <Badge type="warning" text="Local CSV file" /> <Badge type="info" text="MongoDB" /> <Badge type="danger" text="Oracle" /> <Badge type="tip" text="SAP Hana" /> <Badge type="warning" text="SQLite" /> <Badge type="info" text="MySQL" /> <Badge type="danger" text="Google Sheets" /> <Badge type="tip" text="Notion" /> <Badge type="warning" text="Shopify" />
+<Badge type="warning" text="CrateDB" /> <Badge type="info" text="Postgres" /> <Badge type="danger" text="BigQuery" /> <Badge type="tip" text="Snowflake" /> <Badge type="warning" text="Redshift" /> <Badge type="info" text="Databricks" /> <Badge type="danger" text="DuckDB" /> <Badge type="tip" text="Microsoft SQL Server" /> <Badge type="warning" text="Local CSV file" /> <Badge type="info" text="MongoDB" /> <Badge type="danger" text="Oracle" /> <Badge type="tip" text="SAP Hana" /> <Badge type="warning" text="SQLite" /> <Badge type="info" text="MySQL" /> <Badge type="danger" text="Google Sheets" /> <Badge type="tip" text="Notion" /> <Badge type="warning" text="Shopify" />
 </div>

ingestr-0.13.63/docs/supported-sources/linear.md

@@ -0,0 +1,41 @@
+# Linear
+
+[Linear](https://linear.app/) is a purpose-built tool for planning and building products that provides issue tracking and project management for teams.
+
+ingestr supports Linear as a source.
+
+## URI format
+
+The URI format for Linear is:
+
+```plaintext
+linear://?api_key=<api_key>
+```
+
+URI parameters:
+
+- `api_key`: The API key used for authentication with the Linear API.
+
+## Example usage
+
+Assuming your API key is `lin_api_123`, you can ingest teams into DuckDB using:
+
+```bash
+ingestr ingest
+--source-uri 'linear://?api_key=lin_api_123' \
+--source-table 'teams' \
+--dest-uri duckdb:///linear.duckdb \
+--dest-table 'dest.teams'
+```
+<img alt="linear" src="../media/linear.png"/>
+
+## Tables
+Linear source allows ingesting the following tables::
+
+- `issues`: Fetches all issues from your Linear workspace.
+- `projects`: Fetches project-level data, .
+- `teams`: Fetches information about the teams configured in Linear.
+- `users`: Fetches users from your workspace.
+
+
+Use these as the `--source-table` parameter in the `ingestr ingest` command.

ingestr-0.13.63/docs/supported-sources/zoom.md

@@ -0,0 +1,34 @@
+# Zoom
+
+[Zoom](https://www.zoom.com/) is a video conferencing and collaboration platform.
+
+ingestr supports Zoom as a source.
+
+## Prerequisites
+- A [Zoom Server-to-Server OAuth App](https://developers.zoom.us/docs/internal-apps/s2s-oauth/)
+- Appropriate permissions related to meetings and users must be added in the app's scopes
+- Obtain the `client_id`, `client_secret` and `account_id`  credentials from the app
+
+## URI format
+```plaintext
+zoom://?client_id=<client_id>&client_secret=<client_secret>&account_id=<account_id>
+```
+
+This command copies meetings data from the Zoom source to DuckDB.
+
+```sh
+ingestr ingest \
+  --source-uri 'zoom://?client_id=abc&client_secret=xyz&account_id=123' \
+  --source-table 'meetings' \
+  --dest-uri duckdb:///zoom.duckdb \
+  --dest-table 'dest.meetings'
+```
+
+<img alt="zoom" src="../media/zoom_ingestion.png"/>
+## Tables
+
+Zoom source allows ingesting the following tables:
+
+- `meetings`: Retrieves all valid previous meetings, live meetings, and upcoming scheduled meetings for all users of the given Zoom account.
+
+Use these as the `--source-table` parameter in the `ingestr ingest` command.

ingestr-0.13.63/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.63"

{ingestr-0.13.61 → ingestr-0.13.63}/ingestr/src/factory.py

@@ -48,6 +48,7 @@ from ingestr.src.sources import (
     KafkaSource,
     KinesisSource,
     KlaviyoSource,
+    LinearSource,
     LinkedInAdsSource,
     LocalCsvSource,
     MixpanelSource,
@@ -70,6 +71,7 @@ from ingestr.src.sources import (
     TikTokSource,
     TrustpilotSource,
     ZendeskSource,
+    ZoomSource,
 )
 
 SQL_SOURCE_SCHEMES = [
@@ -162,6 +164,7 @@ class SourceDestinationFactory:
         "appstore": AppleAppStoreSource,
         "gs": GCSSource,
         "linkedinads": LinkedInAdsSource,
+        "linear": LinearSource,
         "applovin": AppLovinSource,
         "applovinmax": ApplovinMaxSource,
         "salesforce": SalesforceSource,
@@ -180,6 +183,7 @@ class SourceDestinationFactory:
         "smartsheet": SmartsheetSource,
         "sftp": SFTPSource,
         "pinterest": PinterestSource,
+        "zoom": ZoomSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr-0.13.63/ingestr/src/linear/__init__.py

@@ -0,0 +1,183 @@
+from typing import Any, Dict, Iterable, Iterator, Optional
+
+import dlt
+import pendulum
+import requests
+
+LINEAR_GRAPHQL_ENDPOINT = "https://api.linear.app/graphql"
+
+
+def _graphql(
+    api_key: str, query: str, variables: Optional[Dict[str, Any]] = None
+) -> Dict[str, Any]:
+    headers = {"Authorization": api_key, "Content-Type": "application/json"}
+    response = requests.post(
+        LINEAR_GRAPHQL_ENDPOINT,
+        json={"query": query, "variables": variables or {}},
+        headers=headers,
+    )
+    response.raise_for_status()
+    payload = response.json()
+    if "errors" in payload:
+        raise ValueError(str(payload["errors"]))
+    return payload["data"]
+
+
+def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
+    cursor: Optional[str] = None
+    while True:
+        data = _graphql(api_key, query, {"cursor": cursor})[root]
+        for item in data["nodes"]:
+            yield item
+        if not data["pageInfo"]["hasNextPage"]:
+            break
+        cursor = data["pageInfo"]["endCursor"]
+
+
+ISSUES_QUERY = """
+query Issues($cursor: String) {
+  issues(first: 50, after: $cursor) {
+    nodes {
+      id
+      title
+      description
+      createdAt
+      updatedAt
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+PROJECTS_QUERY = """
+query Projects($cursor: String) {
+  projects(first: 50, after: $cursor) {
+    nodes {
+      id
+      name
+      description
+      createdAt
+      updatedAt
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+TEAMS_QUERY = """
+query Teams($cursor: String) {
+  teams(first: 50, after: $cursor) {
+    nodes {
+      id
+      name
+      key
+      description
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+USERS_QUERY = """
+query Users($cursor: String) {
+  users(first: 50, after: $cursor) {
+    nodes {
+      id
+      name
+      displayName
+      email
+      createdAt
+      updatedAt
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+
+@dlt.source(name="linear", max_table_nesting=0)
+def linear_source(
+    api_key: str,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None = None,
+) -> Iterable[dlt.sources.DltResource]:
+    @dlt.resource(name="issues", primary_key="id", write_disposition="merge")
+    def issues(
+        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "updatedAt",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[Dict[str, Any]]:
+        if updated_at.last_value:
+            current_start_date = pendulum.parse(updated_at.last_value)
+        else:
+            current_start_date = pendulum.parse(start_date)
+
+        if updated_at.end_value:
+            current_end_date = pendulum.parse(updated_at.end_value)
+        else:
+            current_end_date = pendulum.now(tz="UTC")
+
+        for item in _paginate(api_key, ISSUES_QUERY, "issues"):
+            if pendulum.parse(item["updatedAt"]) >= current_start_date:
+                if pendulum.parse(item["updatedAt"]) <= current_end_date:
+                    yield item
+
+    @dlt.resource(name="projects", primary_key="id", write_disposition="merge")
+    def projects(
+        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "updatedAt",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[Dict[str, Any]]:
+        if updated_at.last_value:
+            current_start_date = pendulum.parse(updated_at.last_value)
+        else:
+            current_start_date = pendulum.parse(start_date)
+
+        if updated_at.end_value:
+            current_end_date = pendulum.parse(updated_at.end_value)
+        else:
+            current_end_date = pendulum.now(tz="UTC")
+
+        for item in _paginate(api_key, PROJECTS_QUERY, "projects"):
+            if pendulum.parse(item["updatedAt"]) >= current_start_date:
+                if pendulum.parse(item["updatedAt"]) <= current_end_date:
+                    yield item
+
+    @dlt.resource(name="teams", primary_key="id", write_disposition="merge")
+    def teams() -> Iterator[Dict[str, Any]]:
+        yield from _paginate(api_key, TEAMS_QUERY, "teams")
+
+    @dlt.resource(name="users", primary_key="id", write_disposition="merge")
+    def users(
+        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "updatedAt",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[Dict[str, Any]]:
+        if updated_at.last_value:
+            current_start_date = pendulum.parse(updated_at.last_value)
+        else:
+            current_start_date = pendulum.parse(start_date)
+
+        if updated_at.end_value:
+            current_end_date = pendulum.parse(updated_at.end_value)
+        else:
+            current_end_date = pendulum.now(tz="UTC")
+
+        for item in _paginate(api_key, USERS_QUERY, "users"):
+            if pendulum.parse(item["updatedAt"]) >= current_start_date:
+                if pendulum.parse(item["updatedAt"]) <= current_end_date:
+                    yield item
+
+    return issues, projects, teams, users

{ingestr-0.13.61 → ingestr-0.13.63}/ingestr/src/sources.py

@@ -2812,7 +2812,7 @@ class IsocPulseSource:
         src = pulse_source(
             token=token[0],
             start_date=start_date.strftime("%Y-%m-%d"),
-            end_date=str(end_date) if end_date else None,
+            end_date=end_date.strftime("%Y-%m-%d") if end_date else None,
             metric=metric,
             opts=opts,
         )
@@ -2851,3 +2851,77 @@ class PinterestSource:
             start_date=start_date,
             end_date=end_date,
         ).with_resources(table)
+
+
+class LinearSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+        api_key = params.get("api_key")
+        if api_key is None:
+            raise MissingValueError("api_key", "Linear")
+
+        if table not in ["issues", "projects", "teams", "users"]:
+            raise UnsupportedResourceError(table, "Linear")
+
+        start_date = kwargs.get("interval_start")
+        if start_date is not None:
+            start_date = ensure_pendulum_datetime(start_date)
+        else:
+            start_date = pendulum.datetime(2020, 1, 1).in_tz("UTC")
+
+        end_date = kwargs.get("interval_end")
+        if end_date is not None:
+            end_date = end_date = ensure_pendulum_datetime(end_date).in_tz("UTC")
+
+        from ingestr.src.linear import linear_source
+
+        return linear_source(
+            api_key=api_key[0],
+            start_date=start_date,
+            end_date=end_date,
+        ).with_resources(table)
+
+
+class ZoomSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed = urlparse(uri)
+        params = parse_qs(parsed.query)
+        client_id = params.get("client_id")
+        client_secret = params.get("client_secret")
+        account_id = params.get("account_id")
+
+        if not (client_id and client_secret and account_id):
+            raise MissingValueError(
+                "client_id/client_secret/account_id",
+                "Zoom",
+            )
+
+        start_date = kwargs.get("interval_start")
+        if start_date is not None:
+            start_date = ensure_pendulum_datetime(start_date)
+        else:
+            start_date = pendulum.datetime(2020, 1, 26).in_tz("UTC")
+
+        end_date = kwargs.get("interval_end")
+        if end_date is not None:
+            end_date = end_date = ensure_pendulum_datetime(end_date).in_tz("UTC")
+
+        from ingestr.src.zoom import zoom_source
+
+        if table not in {"meetings"}:
+            raise UnsupportedResourceError(table, "Zoom")
+
+        return zoom_source(
+            client_id=client_id[0],
+            client_secret=client_secret[0],
+            account_id=account_id[0],
+            start_date=start_date,
+            end_date=end_date,
+        ).with_resources(table)

ingestr-0.13.63/ingestr/src/zoom/__init__.py

@@ -0,0 +1,55 @@
+from typing import Any, Dict, Iterable, Sequence
+
+import dlt
+import pendulum
+from dlt.common.typing import TAnyDateTime, TDataItem
+from dlt.sources import DltResource
+
+from .helpers import ZoomClient
+
+
+@dlt.source(name="zoom", max_table_nesting=0)
+def zoom_source(
+    client_id: str,
+    client_secret: str,
+    account_id: str,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None = None,
+) -> Sequence[DltResource]:
+    """Create a Zoom source with meetings resource for all users in the account."""
+    client = ZoomClient(
+        client_id=client_id,
+        client_secret=client_secret,
+        account_id=account_id,
+    )
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def meetings(
+        datetime: dlt.sources.incremental[TAnyDateTime] = dlt.sources.incremental(
+            "start_time",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date is not None else None,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        if datetime.last_value:
+            start_dt = pendulum.parse(datetime.last_value)
+        else:
+            start_dt = pendulum.parse(start_date)
+
+        if end_date is None:
+            end_dt = pendulum.now("UTC")
+        else:
+            end_dt = pendulum.parse(datetime.end_value)
+        base_params: Dict[str, Any] = {
+            "type": "scheduled",
+            "page_size": 300,
+            "from": start_dt.to_date_string(),
+            "to": end_dt.to_date_string(),
+        }
+        for user in client.get_users():
+            user_id = user["id"]
+            yield from client.get_meetings(user_id, base_params)
+
+    return meetings

ingestr-0.13.63/ingestr/src/zoom/helpers.py

@@ -0,0 +1,76 @@
+import time
+from typing import Any, Dict, Iterator, Optional
+
+from ingestr.src.http_client import create_client
+
+
+class ZoomClient:
+    """Minimal Zoom API client supporting Server-to-Server OAuth."""
+
+    def __init__(
+        self,
+        client_id: Optional[str] = None,
+        client_secret: Optional[str] = None,
+        account_id: Optional[str] = None,
+    ) -> None:
+        self.client_id = client_id
+        self.client_secret = client_secret
+        self.account_id = account_id
+        self.token_expires_at: float = 0
+        self.base_url = "https://api.zoom.us/v2"
+        self.session = create_client()
+        self._refresh_access_token()
+
+    def _refresh_access_token(self) -> None:
+        token_url = "https://zoom.us/oauth/token"
+        auth = (self.client_id, self.client_secret)
+        resp = self.session.post(
+            token_url,
+            params={"grant_type": "account_credentials", "account_id": self.account_id},
+            auth=auth,
+        )
+        resp.raise_for_status()
+        data = resp.json()
+        self.access_token = data.get("access_token")
+        self.token_expires_at = time.time() + data.get("expires_in", 3600)
+
+    def _ensure_token(self) -> None:
+        if self.access_token is None or self.token_expires_at <= time.time():
+            self._refresh_access_token()
+
+    def _headers(self) -> Dict[str, str]:
+        self._ensure_token()
+        return {
+            "Authorization": f"Bearer {self.access_token}",
+            "Accept": "application/json",
+        }
+
+    def get_users(self) -> Iterator[Dict[str, Any]]:
+        url = f"{self.base_url}/users"
+        params = {"page_size": 1000}
+        while True:
+            response = self.session.get(url, headers=self._headers(), params=params)
+            response.raise_for_status()
+            data = response.json()
+            for user in data.get("users", []):
+                yield user
+            token = data.get("next_page_token")
+            if not token:
+                break
+            params["next_page_token"] = token
+
+    def get_meetings(
+        self, user_id: str, params: Dict[str, Any]
+    ) -> Iterator[Dict[str, Any]]:
+        url = f"{self.base_url}/users/{user_id}/meetings"
+        while True:
+            response = self.session.get(url, headers=self._headers(), params=params)
+            response.raise_for_status()
+            data = response.json()
+            for item in data.get("meetings", []):
+                item["zoom_user_id"] = user_id
+                yield item
+            token = data.get("next_page_token")
+            if not token:
+                break
+            params["next_page_token"] = token

{ingestr-0.13.61 → ingestr-0.13.63}/pyproject.toml

@@ -89,6 +89,8 @@ exclude = [
     'src/asana_source/.*',
     'src/tiktok_ads/.*',
     'src/pipedrive/.*',
+    'src/linear/.*',
+    'src/zoom/.*',
   ]
 
 [[tool.mypy.overrides]]
@@ -116,6 +118,8 @@ module = [
   "ingestr.src.github.*",
   "ingestr.src.clickhouse.*",
   "ingestr.src.pipedrive.*",
+  "ingestr.src.linear.*",
+  "ingestr.src.zoom.*",
 ]
 follow_imports = "skip"
 

ingestr-0.13.61/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.61"