ingestr 0.13.2__py3-none-any.whl → 0.14.104__py3-none-any.whl

ingestr/src/wise/__init__.py

@@ -0,0 +1,68 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from .client import WiseClient
+
+
+@dlt.source(max_table_nesting=0)
+def wise_source(
+    api_key: str,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None = None,
+) -> Iterable[DltResource]:
+    client = WiseClient(api_key)
+
+    # List of all profiles belonging to user.
+    @dlt.resource(write_disposition="merge", name="profiles", primary_key="id")
+    def profiles() -> Iterable[TDataItem]:
+        yield from client.fetch_profiles()
+
+    # List transfers for a profile.
+    @dlt.resource(write_disposition="merge", name="transfers", primary_key="id")
+    def transfers(
+        profiles=profiles,
+        datetime=dlt.sources.incremental(
+            "created",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ):
+        if datetime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = datetime.end_value
+
+        start_dt = datetime.last_value
+
+        for profile in profiles:
+            yield from client.fetch_transfers(profile["id"], start_dt, end_dt)
+
+    # Retrieve the user's multi-currency account balance accounts. It returns all balance accounts the profile has.
+    @dlt.resource(write_disposition="merge", name="balances", primary_key="id")
+    def balances(
+        profiles=profiles,
+        datetime=dlt.sources.incremental(
+            "modificationTime",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        if datetime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = datetime.end_value
+
+        start_dt = datetime.last_value
+
+        for profile in profiles:
+            yield from client.fetch_balances(profile["id"], start_dt, end_dt)
+
+    return profiles, transfers, balances

ingestr/src/wise/client.py

@@ -0,0 +1,63 @@
+from typing import Iterable
+
+import pendulum
+from dlt.sources.helpers.requests import Client
+
+
+class WiseClient:
+    BASE_URL = "https://api.transferwise.com"
+
+    def __init__(self, api_key: str) -> None:
+        self.session = Client(raise_for_status=False).session
+        self.session.headers.update({"Authorization": f"Bearer {api_key}"})
+
+    # https://docs.wise.com/api-docs/api-reference/profile#list-profiles
+    def fetch_profiles(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v2/profiles"
+        resp = self.session.get(url)
+        resp.raise_for_status()
+        for profile in resp.json():
+            yield profile
+
+    # https://docs.wise.com/api-docs/api-reference/transfer#list-transfers
+    def fetch_transfers(
+        self, profile_id: str, start_time=pendulum.DateTime, end_time=pendulum.DateTime
+    ):
+        offset = 0
+
+        while True:
+            data = self.session.get(
+                f"{self.BASE_URL}/v1/transfers",
+                params={
+                    "profile": profile_id,
+                    "createdDateStart": start_time.to_date_string(),
+                    "createdDateEnd": end_time.to_date_string(),
+                    "limit": 100,
+                    "offset": offset,
+                },
+            )
+            response_data = data.json()
+
+            if not response_data or len(response_data) == 0:
+                break
+
+            for transfer in response_data:
+                transfer["created"] = pendulum.parse(transfer["created"])
+
+                yield transfer
+            offset += 100
+
+    # https://docs.wise.com/api-docs/api-reference/balance#list
+    def fetch_balances(
+        self, profile_id: str, start_time=pendulum.DateTime, end_time=pendulum.DateTime
+    ) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v4/profiles/{profile_id}/balances"
+        resp = self.session.get(url, params={"types": "STANDARD,SAVINGS"})
+        resp.raise_for_status()
+        for balance in resp.json():
+            balance["modificationTime"] = pendulum.parse(balance["modificationTime"])
+            if (
+                balance["modificationTime"] > start_time
+                and balance["modificationTime"] < end_time
+            ):
+                yield balance

ingestr/src/zoom/__init__.py

@@ -0,0 +1,99 @@
+from typing import Any, Dict, Iterable, Sequence
+
+import dlt
+import pendulum
+from dlt.common.typing import TAnyDateTime, TDataItem
+from dlt.sources import DltResource
+
+from .helpers import ZoomClient
+
+
+@dlt.source(name="zoom", max_table_nesting=0)
+def zoom_source(
+    client_id: str,
+    client_secret: str,
+    account_id: str,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None = None,
+) -> Sequence[DltResource]:
+    """Create a Zoom source with meetings resource for all users in the account."""
+    client = ZoomClient(
+        client_id=client_id,
+        client_secret=client_secret,
+        account_id=account_id,
+    )
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def meetings(
+        datetime: dlt.sources.incremental[TAnyDateTime] = dlt.sources.incremental(
+            "start_time",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date is not None else None,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        if datetime.last_value:
+            start_dt = pendulum.parse(datetime.last_value)
+        else:
+            start_dt = pendulum.parse(start_date)
+
+        if end_date is None:
+            end_dt = pendulum.now("UTC")
+        else:
+            end_dt = pendulum.parse(datetime.end_value)
+
+        base_params: Dict[str, Any] = {
+            "type": "scheduled",
+            "page_size": 300,
+            "from": start_dt.to_date_string(),
+            "to": end_dt.to_date_string(),
+        }
+
+        for user in client.get_users():
+            user_id = user["id"]
+            yield from client.get_meetings(user_id, base_params)
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def users() -> Iterable[TDataItem]:
+        yield from client.get_users()
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def participants(
+        datetime: dlt.sources.incremental[TAnyDateTime] = dlt.sources.incremental(
+            "join_time",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date is not None else None,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        if datetime.last_value:
+            start_dt = pendulum.parse(datetime.last_value)
+        else:
+            start_dt = pendulum.parse(start_date)
+
+        if end_date is None:
+            end_dt = pendulum.now("UTC")
+        else:
+            end_dt = pendulum.parse(datetime.end_value)
+
+        participant_params: Dict[str, Any] = {
+            "page_size": 300,
+        }
+        meeting_params = {
+            "type": "previous_meetings",
+            "page_size": 300,
+        }
+        for user in client.get_users():
+            user_id = user["id"]
+            for meeting in client.get_meetings(user_id=user_id, params=meeting_params):
+                meeting_id = meeting["id"]
+                yield from client.get_participants(
+                    meeting_id=meeting_id,
+                    params=participant_params,
+                    start_date=start_dt,
+                    end_date=end_dt,
+                )
+
+    return meetings, users, participants

ingestr/src/zoom/helpers.py

@@ -0,0 +1,102 @@
+import time
+from typing import Any, Dict, Iterator, Optional
+
+import pendulum
+
+from ingestr.src.http_client import create_client
+
+
+class ZoomClient:
+    """Minimal Zoom API client supporting Server-to-Server OAuth."""
+
+    def __init__(
+        self,
+        client_id: Optional[str] = None,
+        client_secret: Optional[str] = None,
+        account_id: Optional[str] = None,
+    ) -> None:
+        self.client_id = client_id
+        self.client_secret = client_secret
+        self.account_id = account_id
+        self.token_expires_at: float = 0
+        self.base_url = "https://api.zoom.us/v2"
+        self.session = create_client()
+        self._refresh_access_token()
+
+    def _refresh_access_token(self) -> None:
+        token_url = "https://zoom.us/oauth/token"
+        auth = (self.client_id, self.client_secret)
+        resp = self.session.post(
+            token_url,
+            params={"grant_type": "account_credentials", "account_id": self.account_id},
+            auth=auth,
+        )
+        resp.raise_for_status()
+        data = resp.json()
+        self.access_token = data.get("access_token")
+        self.token_expires_at = time.time() + data.get("expires_in", 3600)
+
+    def _ensure_token(self) -> None:
+        if self.access_token is None or self.token_expires_at <= time.time():
+            self._refresh_access_token()
+
+    def _headers(self) -> Dict[str, str]:
+        self._ensure_token()
+        return {
+            "Authorization": f"Bearer {self.access_token}",
+            "Accept": "application/json",
+        }
+
+    def get_users(self) -> Iterator[Dict[str, Any]]:
+        url = f"{self.base_url}/users"
+
+        params = {"page_size": 1000}
+        while True:
+            response = self.session.get(url, headers=self._headers(), params=params)
+            response.raise_for_status()
+            data = response.json()
+            for user in data.get("users", []):
+                yield user
+            token = data.get("next_page_token")
+            if not token:
+                break
+            params["next_page_token"] = token
+
+    # https://developers.zoom.us/docs/api/rest/reference/zoom-api/methods/#operation/meetings
+    def get_meetings(
+        self, user_id: str, params: Dict[str, Any]
+    ) -> Iterator[Dict[str, Any]]:
+        url = f"{self.base_url}/users/{user_id}/meetings"
+        while True:
+            response = self.session.get(url, headers=self._headers(), params=params)
+            response.raise_for_status()
+            data = response.json()
+            for item in data.get("meetings", []):
+                item["zoom_user_id"] = user_id
+                yield item
+            token = data.get("next_page_token")
+            if not token:
+                break
+            params["next_page_token"] = token
+
+    # https://developers.zoom.us/docs/api/rest/reference/zoom-api/methods/#operation/reportMeetingParticipants
+    def get_participants(
+        self,
+        meeting_id: str,
+        params: Dict[str, Any],
+        start_date: pendulum.DateTime,
+        end_date: pendulum.DateTime,
+    ) -> Iterator[Dict[str, Any]]:
+        url = f"{self.base_url}/report/meetings/{meeting_id}/participants"
+        while True:
+            response = self.session.get(url, headers=self._headers(), params=params)
+            response.raise_for_status()
+            data = response.json()
+            for item in data.get("participants", []):
+                join_time = pendulum.parse(item["join_time"])
+                if join_time >= start_date and join_time <= end_date:
+                    yield item
+            token = data.get("next_page_token")
+            if not token:
+                break
+            params["next_page_token"] = token

ingestr/tests/unit/test_smartsheets.py

@@ -0,0 +1,133 @@
+import sys
+import unittest
+from unittest.mock import patch
+
+import smartsheet  # type: ignore
+from smartsheet.models import Cell, Column, Row, Sheet  # type: ignore
+
+from ingestr.src.smartsheets import _get_sheet_data, smartsheet_source
+
+
+def pp(x):
+    print(x, file=sys.stderr)
+
+
+class TestSmartsheetSource(unittest.TestCase):
+    @patch("ingestr.src.smartsheets.smartsheet.Smartsheet")
+    def test_smartsheet_source_success(self, mock_smartsheet_client):
+        # Mock Smartsheet client and its methods
+        mock_client_instance = mock_smartsheet_client.return_value
+
+        # Mock sheet details response
+        mock_sheet_details = Sheet(
+            {
+                "id": 123,
+                "name": "Test Sheet 1",
+                "columns": [
+                    Column(
+                        {"id": 1, "title": "Col A", "type": "TEXT_NUMBER", "index": 0}
+                    ),
+                    Column(
+                        {"id": 2, "title": "Col B", "type": "TEXT_NUMBER", "index": 1}
+                    ),
+                ],
+                "rows": [
+                    Row(
+                        {
+                            "id": 101,
+                            "sheetId": 123,
+                            "cells": [
+                                Cell({"columnId": 1, "value": "r1c1"}),
+                                Cell({"columnId": 2, "value": "r1c2"}),
+                            ],
+                        }
+                    ),
+                    Row(
+                        {
+                            "id": 102,
+                            "sheetId": 123,
+                            "cells": [
+                                Cell({"columnId": 1, "value": "r2c1"}),
+                                Cell({"columnId": 2, "value": "r2c2"}),
+                            ],
+                        }
+                    ),
+                ],
+            }
+        )
+        mock_client_instance.Sheets.get_sheet.return_value = mock_sheet_details
+
+        resource = smartsheet_source(access_token="test_token", sheet_id="123")
+        data = list(resource)
+        self.assertEqual(len(data), 2)
+        self.assertEqual(data[0], {"_row_id": 101, "Col A": "r1c1", "Col B": "r1c2"})
+        self.assertEqual(data[1], {"_row_id": 102, "Col A": "r2c1", "Col B": "r2c2"})
+
+        mock_smartsheet_client.assert_called_once_with("test_token")
+        mock_client_instance.Sheets.get_sheet.assert_any_call(
+            123, include=["objectValue"]
+        )  # for resource name
+        mock_client_instance.Sheets.get_sheet.assert_any_call(
+            123
+        )  # for _get_sheet_data
+
+    @patch("ingestr.src.smartsheets.smartsheet.Smartsheet")
+    def test_smartsheet_source_api_error(self, mock_smartsheet_client):
+        mock_client_instance = mock_smartsheet_client.return_value
+        mock_client_instance.Sheets.get_sheet.side_effect = (
+            smartsheet.exceptions.ApiError("API Error", 500)
+        )
+
+        with self.assertRaises(smartsheet.exceptions.ApiError):
+            source = smartsheet_source(access_token="test_token", sheet_id="123")
+            # Consume the generator to trigger the API call
+            list(source)
+
+    def test_get_sheet_data(self):
+        mock_sheet = Sheet(
+            {
+                "id": 456,
+                "name": "Data Sheet",
+                "columns": [
+                    Column(
+                        {"id": 10, "title": "ID", "type": "TEXT_NUMBER", "index": 0}
+                    ),
+                    Column(
+                        {"id": 20, "title": "Value", "type": "TEXT_NUMBER", "index": 1}
+                    ),
+                ],
+                "rows": [
+                    Row(
+                        {
+                            "id": 201,
+                            "sheetId": 456,
+                            "cells": [
+                                Cell({"columnId": 10, "value": 1}),
+                                Cell({"columnId": 20, "value": "Alpha"}),
+                            ],
+                        }
+                    ),
+                    Row(
+                        {
+                            "id": 202,
+                            "sheetId": 456,
+                            "cells": [
+                                Cell({"columnId": 10, "value": 2}),
+                                Cell({"columnId": 20, "value": "Beta"}),
+                            ],
+                        }
+                    ),
+                ],
+            }
+        )
+
+        data_generator = _get_sheet_data(mock_sheet)
+        data = list(data_generator)
+
+        self.assertEqual(len(data), 2)
+        self.assertEqual(data[0], {"_row_id": 201, "ID": 1, "Value": "Alpha"})
+        self.assertEqual(data[1], {"_row_id": 202, "ID": 2, "Value": "Beta"})
+
+
+if __name__ == "__main__":
+    unittest.main()