ingestr 0.13.63__tar.gz → 0.13.65__tar.gz

{ingestr-0.13.63 → ingestr-0.13.65}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.63
+Version: 0.13.65
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.63 → ingestr-0.13.65}/docs/supported-sources/mssql.md

@@ -28,3 +28,21 @@ URI parameters:
 - `TrustServerCertificate`: whether to trust the server certificate
 
 The same URI structure can be used both for sources and destinations. You can read more about SQLAlchemy's SQL Server dialect [here](https://docs.sqlalchemy.org/en/20/core/engines.html#microsoft-sql-server).
+
+## Tips & Tricks
+
+If you're using Azure SQL Server, you can use `az cli` to generate access tokens to connect to SQL server. 
+
+Set the password to your token and the `Authentication` parameter to `ActiveDirectoryAccessToken`
+::: code-group
+
+```sh [token-auth-example.sh]
+USER=$(az account show --query user.name -o tsv)
+TOKEN=$(az account get-access-token --resource https://database.windows.net/ --query accessToken -o tsv)
+ingestr ingest \
+    --source-uri "mssql://$USER:$TOKEN@<server>.database.windows.net/<database>?Authentication=ActiveDirectoryAccessToken" \
+    --source-table "dbo.example" \
+    --dest-uri "duckdb:///example.db" \
+    --dest-table "dbo.example" \
+```
+:::

ingestr-0.13.65/docs/supported-sources/zoom.md

@@ -0,0 +1,47 @@
+# Zoom
+
+[Zoom](https://www.zoom.com/) is a video conferencing and collaboration platform.
+
+ingestr supports Zoom as a source.
+
+## Prerequisites
+- A [Zoom Server-to-Server OAuth App](https://developers.zoom.us/docs/internal-apps/s2s-oauth/)
+- Appropriate permissions related to meetings, users and participants must be added in the app's scopes
+`user:read,user:write,user:read:admin,user:write:admin`
+- Obtain the `client_id`, `client_secret` and `account_id`  credentials from the app
+
+## URI format
+```plaintext
+zoom://?client_id=<client_id>&client_secret=<client_secret>&account_id=<account_id>
+```
+
+This command copies meetings data from the Zoom source to DuckDB.
+
+```sh
+ingestr ingest \
+  --source-uri 'zoom://?client_id=abc&client_secret=xyz&account_id=123' \
+  --source-table 'meetings' \
+  --dest-uri duckdb:///zoom.duckdb \
+  --dest-table 'dest.meetings'
+```
+
+<img alt="zoom" src="../media/zoom_ingestion.png"/>
+## Tables
+
+Zoom source allows ingesting the following tables:
+- `meetings`: Retrieve all valid previous meetings, live meetings, and upcoming scheduled meetings for all users in the given Zoom account. 
+    - Permissions required: meeting:read:admin,meeting:read
+    - Granular permissions: meeting:read:list_meetings,meeting:read:list_meetings:admin
+
+- `users`: Retrieve a list of users in your account.
+    - Permissions required: user:read, user:write, user:read:admin, user:write:admin.
+    - Granular permissions: user:read:list_users:admin.
+    - Prerequisites: A Pro or higher plan.
+
+- `participants`: Return a report of a past meeting that had participants, including the host.
+ It only returns data for meetings within the last 6 months.
+  - Permissions required: report:read:admin.
+  - Granular permissions: report:read:list_meeting_participants:admin.
+  - Prerequisites: A Pro or higher plan.
+
+Use these as the `--source-table` parameter in the `ingestr ingest` command.

{ingestr-0.13.63 → ingestr-0.13.65}/ingestr/src/attio/__init__.py

@@ -20,13 +20,15 @@ def attio_source(
             "created_at": {"data_type": "timestamp", "partition": True},
         },
     )
+    # https://docs.attio.com/rest-api/endpoint-reference/objects/list-objects - does not support pagination
     def fetch_objects() -> Iterator[dict]:
         if len(params) != 0:
             raise ValueError("Objects table must be in the format `objects`")
 
         path = "objects"
-        yield attio_client.fetch_data(path, "get")
+        yield attio_client.fetch_all(path, "get")
 
+    # https://docs.attio.com/rest-api/endpoint-reference/records/list-records
     @dlt.resource(
         name="records",
         write_disposition="replace",
@@ -39,12 +41,12 @@ def attio_source(
             raise ValueError(
                 "Records table must be in the format `records:{object_api_slug}`"
             )
-
         object_id = params[0]
         path = f"objects/{object_id}/records/query"
 
-        yield attio_client.fetch_data(path, "post")
+        yield attio_client.fetch_paginated(path, "post")
 
+    # https://docs.attio.com/rest-api/endpoint-reference/lists/list-all-lists -- does not support pagination
     @dlt.resource(
         name="lists",
         write_disposition="replace",
@@ -54,8 +56,9 @@ def attio_source(
     )
     def fetch_lists() -> Iterator[dict]:
         path = "lists"
-        yield attio_client.fetch_data(path, "get")
+        yield attio_client.fetch_all(path, "get")
 
+    # https://docs.attio.com/rest-api/endpoint-reference/entries/list-entries
     @dlt.resource(
         name="list_entries",
         write_disposition="replace",
@@ -70,7 +73,7 @@ def attio_source(
             )
         path = f"lists/{params[0]}/entries/query"
 
-        yield attio_client.fetch_data(path, "post")
+        yield attio_client.fetch_paginated(path, "post")
 
     @dlt.resource(
         name="all_list_entries",
@@ -85,10 +88,10 @@ def attio_source(
                 "All list entries table must be in the format `all_list_entries:{object_api_slug}`"
             )
         path = "lists"
-        for lst in attio_client.fetch_data(path, "get"):
+        for lst in attio_client.fetch_all(path, "get"):
             if params[0] in lst["parent_object"]:
                 path = f"lists/{lst['id']['list_id']}/entries/query"
-                yield from attio_client.fetch_data(path, "post")
+                yield from attio_client.fetch_paginated(path, "post")
 
     return (
         fetch_objects,

{ingestr-0.13.63 → ingestr-0.13.65}/ingestr/src/attio/helpers.py

@@ -10,42 +10,53 @@ class AttioClient:
         }
         self.client = create_client()
 
-    def fetch_data(self, path: str, method: str, limit: int = 1000, params=None):
+    def fetch_paginated(self, path: str, method: str, limit: int = 1000, params=None):
         url = f"{self.base_url}/{path}"
         if params is None:
             params = {}
         offset = 0
         while True:
-            query_params = {**params, "limit": limit, "offset": offset}
+            query_params = {"limit": limit, "offset": offset, **params}
             if method == "get":
                 response = self.client.get(
                     url, headers=self.headers, params=query_params
                 )
             else:
-                response = self.client.post(
-                    url, headers=self.headers, params=query_params
-                )
+                json_body = {**params, "limit": limit, "offset": offset}
+                response = self.client.post(url, headers=self.headers, json=json_body)
 
             if response.status_code != 200:
                 raise Exception(f"HTTP {response.status_code} error: {response.text}")
 
             response_data = response.json()
             if "data" not in response_data:
-                print(f"API Response: {response_data}")
                 raise Exception(
                     "Attio API returned a response without the expected data"
                 )
 
             data = response_data["data"]
-
             for item in data:
                 flat_item = flatten_item(item)
                 yield flat_item
-
             if len(data) < limit:
                 break
+
             offset += limit
 
+    def fetch_all(self, path: str, method: str = "get", params=None):
+        url = f"{self.base_url}/{path}"
+        params = params or {}
+
+        if method == "get":
+            response = self.client.get(url, headers=self.headers, params=params)
+        else:
+            response = self.client.post(url, headers=self.headers, json=params)
+
+        response.raise_for_status()
+        data = response.json().get("data", [])
+        for item in data:
+            yield flatten_item(item)
+
 
 def flatten_item(item: dict) -> dict:
     if "id" in item:

ingestr-0.13.65/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.65"

{ingestr-0.13.63 → ingestr-0.13.65}/ingestr/src/destinations.py

@@ -1,19 +1,31 @@
 import abc
 import base64
 import csv
+import datetime
 import json
 import os
 import shutil
+import struct
 import tempfile
 from urllib.parse import parse_qs, quote, urlparse
 
 import dlt
 import dlt.destinations.impl.filesystem.filesystem
 from dlt.common.configuration.specs import AwsCredentials
+from dlt.common.destination.capabilities import DestinationCapabilitiesContext
+from dlt.common.schema import Schema
 from dlt.common.storages.configuration import FileSystemCredentials
 from dlt.destinations.impl.clickhouse.configuration import (
     ClickHouseCredentials,
 )
+from dlt.destinations.impl.mssql.configuration import MsSqlClientConfiguration
+from dlt.destinations.impl.mssql.mssql import (
+    HINT_TO_MSSQL_ATTR,
+    MsSqlJobClient,
+)
+from dlt.destinations.impl.mssql.sql_client import (
+    PyOdbcMsSqlClient,
+)
 
 from ingestr.src.errors import MissingValueError
 from ingestr.src.loader import load_dlt_file
@@ -143,9 +155,89 @@ class DuckDBDestination(GenericSqlDestination):
         return dlt.destinations.duckdb(uri, **kwargs)
 
 
+def handle_datetimeoffset(dto_value: bytes) -> datetime.datetime:
+    # ref: https://github.com/mkleehammer/pyodbc/issues/134#issuecomment-281739794
+    tup = struct.unpack(
+        "<6hI2h", dto_value
+    )  # e.g., (2017, 3, 16, 10, 35, 18, 500000000, -6, 0)
+    return datetime.datetime(
+        tup[0],
+        tup[1],
+        tup[2],
+        tup[3],
+        tup[4],
+        tup[5],
+        tup[6] // 1000,
+        datetime.timezone(datetime.timedelta(hours=tup[7], minutes=tup[8])),
+    )
+
+
+class OdbcMsSqlClient(PyOdbcMsSqlClient):
+    SQL_COPT_SS_ACCESS_TOKEN = 1256
+    SKIP_CREDENTIALS = {"PWD", "AUTHENTICATION", "UID"}
+
+    def open_connection(self):
+        cfg = self.credentials._get_odbc_dsn_dict()
+        if (
+            cfg.get("AUTHENTICATION", "").strip().lower()
+            != "activedirectoryaccesstoken"
+        ):
+            return super().open_connection()
+
+        import pyodbc  # type: ignore
+
+        dsn = ";".join(
+            [f"{k}={v}" for k, v in cfg.items() if k not in self.SKIP_CREDENTIALS]
+        )
+
+        self._conn = pyodbc.connect(
+            dsn,
+            timeout=self.credentials.connect_timeout,
+            attrs_before={
+                self.SQL_COPT_SS_ACCESS_TOKEN: self.serialize_token(cfg["PWD"]),
+            },
+        )
+
+        # https://github.com/mkleehammer/pyodbc/wiki/Using-an-Output-Converter-function
+        self._conn.add_output_converter(-155, handle_datetimeoffset)
+        self._conn.autocommit = True
+        return self._conn
+
+    def serialize_token(self, token):
+        # https://github.com/mkleehammer/pyodbc/issues/228#issuecomment-494773723
+        encoded = token.encode("utf_16_le")
+        return struct.pack("<i", len(encoded)) + encoded
+
+
+class MsSqlClient(MsSqlJobClient):
+    def __init__(
+        self,
+        schema: Schema,
+        config: MsSqlClientConfiguration,
+        capabilities: DestinationCapabilitiesContext,
+    ) -> None:
+        sql_client = OdbcMsSqlClient(
+            config.normalize_dataset_name(schema),
+            config.normalize_staging_dataset_name(schema),
+            config.credentials,
+            capabilities,
+        )
+        super(MsSqlJobClient, self).__init__(schema, config, sql_client)
+        self.config: MsSqlClientConfiguration = config
+        self.sql_client = sql_client
+        self.active_hints = HINT_TO_MSSQL_ATTR if self.config.create_indexes else {}
+        self.type_mapper = capabilities.get_type_mapper()
+
+
+class MsSqlDestImpl(dlt.destinations.mssql):
+    @property
+    def client_class(self):
+        return MsSqlClient
+
+
 class MsSQLDestination(GenericSqlDestination):
     def dlt_dest(self, uri: str, **kwargs):
-        return dlt.destinations.mssql(credentials=uri, **kwargs)
+        return MsSqlDestImpl(credentials=uri, **kwargs)
 
 
 class DatabricksDestination(GenericSqlDestination):

{ingestr-0.13.63 → ingestr-0.13.65}/ingestr/src/factory.py

@@ -79,6 +79,7 @@ SQL_SOURCE_SCHEMES = [
     "crate",
     "duckdb",
     "mssql",
+    "mssql+pyodbc",
     "mysql",
     "mysql+pymysql",
     "mysql+mysqlconnector",

{ingestr-0.13.63 → ingestr-0.13.65}/ingestr/src/linear/__init__.py

@@ -1,38 +1,9 @@
-from typing import Any, Dict, Iterable, Iterator, Optional
+from typing import Any, Dict, Iterable, Iterator
 
 import dlt
 import pendulum
-import requests
-
-LINEAR_GRAPHQL_ENDPOINT = "https://api.linear.app/graphql"
-
-
-def _graphql(
-    api_key: str, query: str, variables: Optional[Dict[str, Any]] = None
-) -> Dict[str, Any]:
-    headers = {"Authorization": api_key, "Content-Type": "application/json"}
-    response = requests.post(
-        LINEAR_GRAPHQL_ENDPOINT,
-        json={"query": query, "variables": variables or {}},
-        headers=headers,
-    )
-    response.raise_for_status()
-    payload = response.json()
-    if "errors" in payload:
-        raise ValueError(str(payload["errors"]))
-    return payload["data"]
-
-
-def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
-    cursor: Optional[str] = None
-    while True:
-        data = _graphql(api_key, query, {"cursor": cursor})[root]
-        for item in data["nodes"]:
-            yield item
-        if not data["pageInfo"]["hasNextPage"]:
-            break
-        cursor = data["pageInfo"]["endCursor"]
 
+from .helpers import  _paginate, _normalize_issue, _normalize_team
 
 ISSUES_QUERY = """
 query Issues($cursor: String) {
@@ -43,6 +14,17 @@ query Issues($cursor: String) {
       description
       createdAt
       updatedAt
+      creator { id }
+      assignee { id}
+      state { id}
+      labels { nodes { id } }
+      cycle { id}
+      project { id }
+      subtasks: children { nodes { id title } }
+      comments(first: 250) { nodes { id body } }
+      priority
+      attachments { nodes { id } }
+      subscribers { nodes { id } }
     }
     pageInfo { hasNextPage endCursor }
   }
@@ -58,6 +40,10 @@ query Projects($cursor: String) {
       description
       createdAt
       updatedAt
+      health
+      priority
+      targetDate
+      lead { id }
     }
     pageInfo { hasNextPage endCursor }
   }
@@ -72,6 +58,11 @@ query Teams($cursor: String) {
       name
       key
       description
+      updatedAt
+      createdAt
+      memberships { nodes { id } }
+      members { nodes { id } }
+      projects { nodes { id } }
     }
     pageInfo { hasNextPage endCursor }
   }
@@ -124,7 +115,7 @@ def linear_source(
         for item in _paginate(api_key, ISSUES_QUERY, "issues"):
             if pendulum.parse(item["updatedAt"]) >= current_start_date:
                 if pendulum.parse(item["updatedAt"]) <= current_end_date:
-                    yield item
+                    yield _normalize_issue(item)
 
     @dlt.resource(name="projects", primary_key="id", write_disposition="merge")
     def projects(
@@ -152,8 +143,29 @@ def linear_source(
                     yield item
 
     @dlt.resource(name="teams", primary_key="id", write_disposition="merge")
-    def teams() -> Iterator[Dict[str, Any]]:
-        yield from _paginate(api_key, TEAMS_QUERY, "teams")
+    def teams( updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "updatedAt",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
+            range_start="closed",
+            range_end="closed",
+        ),) -> Iterator[Dict[str, Any]]:
+        print(start_date)
+        if updated_at.last_value:
+            current_start_date = pendulum.parse(updated_at.last_value)
+        else:
+            current_start_date = pendulum.parse(start_date)
+        print(current_start_date)
+
+        if updated_at.end_value:
+            current_end_date = pendulum.parse(updated_at.end_value)
+        else:
+            current_end_date = pendulum.now(tz="UTC")
+
+        for item in _paginate(api_key, TEAMS_QUERY, "teams"):
+            if pendulum.parse(item["updatedAt"]) >= current_start_date:
+                if pendulum.parse(item["updatedAt"]) <= current_end_date:
+                    yield _normalize_team(item)
 
     @dlt.resource(name="users", primary_key="id", write_disposition="merge")
     def users(

ingestr-0.13.65/ingestr/src/linear/helpers.py

@@ -0,0 +1,60 @@
+import json
+from typing import Any, Dict, Iterator, Optional
+
+import requests
+
+LINEAR_GRAPHQL_ENDPOINT = "https://api.linear.app/graphql"
+
+def _graphql(
+    api_key: str, query: str, variables: Optional[Dict[str, Any]] = None
+) -> Dict[str, Any]:
+    headers = {"Authorization": api_key, "Content-Type": "application/json"}
+    response = requests.post(
+        LINEAR_GRAPHQL_ENDPOINT,
+        json={"query": query, "variables": variables or {}},
+        headers=headers,
+    )
+    response.raise_for_status()
+    payload = response.json()
+    if "errors" in payload:
+        raise ValueError(str(payload["errors"]))
+    return payload["data"]
+
+def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
+    cursor: Optional[str] = None
+    while True:
+        data = _graphql(api_key, query, {"cursor": cursor})[root]
+        for item in data["nodes"]:
+            yield item
+        if not data["pageInfo"]["hasNextPage"]:
+            break
+        cursor = data["pageInfo"]["endCursor"]
+
+def _normalize_issue(item: Dict[str, Any]) -> Dict[str, Any]:
+    field_mapping = {
+        "assignee": "assignee_id",
+        "creator": "creator_id",
+        "state": "state_id",
+        "cycle": "cycle_id",
+        "project": "project_id",
+    }
+    for key, value in field_mapping.items():
+        if item.get(key):
+            item[value] = item[key]["id"]
+            del item[key]
+        else:
+            item[value] = None
+            del item[key]
+    json_fields = ["comments", "subscribers", "attachments", "labels", "subtasks","projects", "memberships", "members"]
+    for field in json_fields:
+        if item.get(field):
+            item[f"{field}"] = item[field].get("nodes", [])
+                
+    return item
+
+def _normalize_team(item: Dict[str, Any]) -> Dict[str, Any]:
+    json_fields = ["memberships", "members", "projects"]
+    for field in json_fields:
+        if item.get(field):
+            item[f"{field}"] = item[field].get("nodes", [])
+    return item

{ingestr-0.13.63 → ingestr-0.13.65}/ingestr/src/sources.py

@@ -258,8 +258,53 @@ class SqlSource:
             # override the query adapters, the only one we want is the one here in the case of custom queries
             query_adapters = [custom_query_variable_subsitution(query_value, kwargs)]
 
+        credentials = ConnectionStringCredentials(uri)
+        if uri.startswith("mssql://"):
+            parsed_uri = urlparse(uri)
+            params = parse_qs(parsed_uri.query)
+            params = {k.lower(): v for k, v in params.items()}
+            if params.get("authentication") == ["ActiveDirectoryAccessToken"]:
+                import pyodbc  # type: ignore
+                from sqlalchemy import create_engine
+
+                from ingestr.src.destinations import (
+                    OdbcMsSqlClient,
+                    handle_datetimeoffset,
+                )
+
+                cfg = {
+                    "DRIVER": params.get("driver", ["ODBC Driver 18 for SQL Server"])[
+                        0
+                    ],
+                    "SERVER": f"{parsed_uri.hostname},{parsed_uri.port or 1433}",
+                    "DATABASE": parsed_uri.path.lstrip("/"),
+                }
+                for k, v in params.items():
+                    if k.lower() not in ["driver", "authentication", "connect_timeout"]:
+                        cfg[k.upper()] = v[0]
+
+                token = OdbcMsSqlClient.serialize_token(None, parsed_uri.password)  # type: ignore[arg-type]
+                dsn = ";".join([f"{k}={v}" for k, v in cfg.items()])
+
+                def creator():
+                    connection = pyodbc.connect(
+                        dsn,
+                        autocommit=True,
+                        timeout=kwargs.get("connect_timeout", 30),
+                        attrs_before={
+                            OdbcMsSqlClient.SQL_COPT_SS_ACCESS_TOKEN: token,
+                        },
+                    )
+                    connection.add_output_converter(-155, handle_datetimeoffset)
+                    return connection
+
+                credentials = create_engine(
+                    "mssql+pyodbc://",
+                    creator=creator,
+                )
+
         builder_res = self.table_builder(
-            credentials=ConnectionStringCredentials(uri),
+            credentials=credentials,
             schema=table_fields.dataset,
             table=table_fields.table,
             incremental=incremental,
@@ -2915,7 +2960,7 @@ class ZoomSource:
 
         from ingestr.src.zoom import zoom_source
 
-        if table not in {"meetings"}:
+        if table not in {"meetings", "users", "participants"}:
             raise UnsupportedResourceError(table, "Zoom")
 
         return zoom_source(

{ingestr-0.13.63 → ingestr-0.13.65}/ingestr/src/zoom/__init__.py

@@ -42,14 +42,58 @@ def zoom_source(
             end_dt = pendulum.now("UTC")
         else:
             end_dt = pendulum.parse(datetime.end_value)
+        
         base_params: Dict[str, Any] = {
             "type": "scheduled",
             "page_size": 300,
             "from": start_dt.to_date_string(),
             "to": end_dt.to_date_string(),
         }
+        
         for user in client.get_users():
             user_id = user["id"]
             yield from client.get_meetings(user_id, base_params)
 
-    return meetings
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def users() -> Iterable[TDataItem]:
+        yield from client.get_users()
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def participants(
+        datetime: dlt.sources.incremental[TAnyDateTime] = dlt.sources.incremental(
+            "join_time",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date is not None else None,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        if datetime.last_value:
+            start_dt = pendulum.parse(datetime.last_value)
+        else:
+            start_dt = pendulum.parse(start_date)
+
+        if end_date is None:
+            end_dt = pendulum.now("UTC")
+        else:
+            end_dt = pendulum.parse(datetime.end_value)
+
+        participant_params: Dict[str, Any] = {
+            "page_size": 300,
+        }
+        meeting_params = {
+            "type": "previous_meetings",
+            "page_size": 300,
+        }
+        for user in client.get_users():
+            user_id = user["id"]
+            for meeting in client.get_meetings(user_id=user_id, params=meeting_params):
+                meeting_id = meeting["id"]
+                yield from client.get_participants(
+                    meeting_id=meeting_id,
+                    params=participant_params,
+                    start_date=start_dt,
+                    end_date=end_dt,
+                )
+
+    return meetings, users, participants