ingestr 0.13.82__tar.gz → 0.13.83__tar.gz

{ingestr-0.13.82 → ingestr-0.13.83}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.82
+Version: 0.13.83
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.82 → ingestr-0.13.83}/docs/.vitepress/config.mjs

@@ -155,6 +155,7 @@ export default defineConfig({
               { text: "Solidgate", link: "/supported-sources/solidgate.md" },
               { text: "Stripe", link: "/supported-sources/stripe.md" },
               { text: "TikTok Ads", link: "/supported-sources/tiktok-ads.md" },
+              { text: "Wise", link: "/supported-sources/wise.md" },
               { text: "Zendesk", link: "/supported-sources/zendesk.md" },
               { text: "Zoom", link: "/supported-sources/zoom.md" },
             ],

ingestr-0.13.83/docs/supported-sources/wise.md

@@ -0,0 +1,49 @@
+# Wise
+
+[Wise](https://wise.com/) is a global financial technology company that provides international money transfers, multi-currency accounts, and business payment solutions.
+
+ingestr supports Wise as a source.
+
+## URI format
+
+```plaintext
+wise://?api_key=<api_token>
+```
+
+URI parameters:
+
+- `api_key`: Wise API token.
+
+### Please follow the steps below to get a Wise API token
+- Go to wise.com and log in.
+- Click on your profile picture.
+- From the dropdown, choose "Settings".
+- In the left-hand menu, click "API Tokens".
+- Click "Create API Token".
+- Give it a name.
+- Click "Create".
+Copy the token and save it securely..
+
+## Example
+
+Copy transfers from Wise into a DuckDB database:
+
+```sh
+ingestr ingest \
+    --source-uri 'wise://?api_key=mytoken' \
+    --source-table transfers \
+    --dest-uri duckdb:///wise.duckdb \
+    --dest-table wise.transfers
+```
+
+## Tables
+
+Wise source allows ingesting the following sources into separate tables:
+| Table           | PK | Inc Key | Inc Strategy | Details                                                                                                                                        |
+| --------------- | ----------- | --------------- | ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- |
+| `profiles`        | `id` | –                | merge               | Retrieves all profiles  associated with the user. |
+| `transfers` | `id` |    `created`             | merge               | Lists all transfers for including status and details associated with the user.                                           |
+| `balances`   | `id` | `modificationTime`     | merge  | Retrieve the user's multi-currency account balance accounts. It returns all balance accounts the profile has.
+
+
+<img alt="wise_ingestion_img" src="../media/wise_ingestion.png" />

ingestr-0.13.83/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.83"

{ingestr-0.13.82 → ingestr-0.13.83}/ingestr/src/factory.py

@@ -73,6 +73,7 @@ from ingestr.src.sources import (
     StripeAnalyticsSource,
     TikTokSource,
     TrustpilotSource,
+    WiseSource,
     ZendeskSource,
     ZoomSource,
 )
@@ -192,6 +193,7 @@ class SourceDestinationFactory:
         "zoom": ZoomSource,
         "clickup": ClickupSource,
         "influxdb": InfluxDBSource,
+        "wise": WiseSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

{ingestr-0.13.82 → ingestr-0.13.83}/ingestr/src/sources.py

@@ -3386,3 +3386,41 @@ class InfluxDBSource:
             start_date=start_date,
             end_date=end_date,
         ).with_resources(table)
+
+
+class WiseSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed = urlparse(uri)
+        params = parse_qs(parsed.query)
+        api_key = params.get("api_key")
+
+        if not api_key:
+            raise MissingValueError("api_key", "Wise")
+
+        if table not in ["profiles", "transfers", "balances"]:
+            raise ValueError(
+                f"Resource '{table}' is not supported for Wise source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+            )
+
+        start_date = kwargs.get("interval_start")
+        if start_date:
+            start_date = ensure_pendulum_datetime(start_date).in_timezone("UTC")
+        else:
+            start_date = pendulum.datetime(2020, 1, 1).in_timezone("UTC")
+
+        end_date = kwargs.get("interval_end")
+        if end_date:
+            end_date = ensure_pendulum_datetime(end_date).in_timezone("UTC")
+        else:
+            end_date = None
+
+        from ingestr.src.wise import wise_source
+
+        return wise_source(
+            api_key=api_key[0],
+            start_date=start_date,
+            end_date=end_date,
+        ).with_resources(table)

ingestr-0.13.83/ingestr/src/wise/__init__.py

@@ -0,0 +1,68 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from .client import WiseClient
+
+
+@dlt.source(max_table_nesting=0)
+def wise_source(
+    api_key: str,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None = None,
+) -> Iterable[DltResource]:
+    client = WiseClient(api_key)
+
+    # List of all profiles belonging to user.
+    @dlt.resource(write_disposition="merge", name="profiles", primary_key="id")
+    def profiles() -> Iterable[TDataItem]:
+        yield from client.fetch_profiles()
+
+    # List transfers for a profile.
+    @dlt.resource(write_disposition="merge", name="transfers", primary_key="id")
+    def transfers(
+        profiles=profiles,
+        datetime=dlt.sources.incremental(
+            "created",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ):
+        if datetime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = datetime.end_value
+
+        start_dt = datetime.last_value
+
+        for profile in profiles:
+            yield from client.fetch_transfers(profile["id"], start_dt, end_dt)
+
+    # Retrieve the user's multi-currency account balance accounts. It returns all balance accounts the profile has.
+    @dlt.resource(write_disposition="merge", name="balances", primary_key="id")
+    def balances(
+        profiles=profiles,
+        datetime=dlt.sources.incremental(
+            "modificationTime",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        if datetime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = datetime.end_value
+
+        start_dt = datetime.last_value
+
+        for profile in profiles:
+            yield from client.fetch_balances(profile["id"], start_dt, end_dt)
+
+    return profiles, transfers, balances

ingestr-0.13.83/ingestr/src/wise/client.py

@@ -0,0 +1,63 @@
+from typing import Iterable
+
+import pendulum
+from dlt.sources.helpers.requests import Client
+
+
+class WiseClient:
+    BASE_URL = "https://api.transferwise.com"
+
+    def __init__(self, api_key: str) -> None:
+        self.session = Client(raise_for_status=False).session
+        self.session.headers.update({"Authorization": f"Bearer {api_key}"})
+
+    # https://docs.wise.com/api-docs/api-reference/profile#list-profiles
+    def fetch_profiles(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v2/profiles"
+        resp = self.session.get(url)
+        resp.raise_for_status()
+        for profile in resp.json():
+            yield profile
+
+    # https://docs.wise.com/api-docs/api-reference/transfer#list-transfers
+    def fetch_transfers(
+        self, profile_id: str, start_time=pendulum.DateTime, end_time=pendulum.DateTime
+    ):
+        offset = 0
+
+        while True:
+            data = self.session.get(
+                f"{self.BASE_URL}/v1/transfers",
+                params={
+                    "profile": profile_id,
+                    "createdDateStart": start_time.to_date_string(),
+                    "createdDateEnd": end_time.to_date_string(),
+                    "limit": 100,
+                    "offset": offset,
+                },
+            )
+            response_data = data.json()
+
+            if not response_data or len(response_data) == 0:
+                break
+
+            for transfer in response_data:
+                transfer["created"] = pendulum.parse(transfer["created"])
+
+                yield transfer
+            offset += 100
+
+    # https://docs.wise.com/api-docs/api-reference/balance#list
+    def fetch_balances(
+        self, profile_id: str, start_time=pendulum.DateTime, end_time=pendulum.DateTime
+    ) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v4/profiles/{profile_id}/balances"
+        resp = self.session.get(url, params={"types": "STANDARD,SAVINGS"})
+        resp.raise_for_status()
+        for balance in resp.json():
+            balance["modificationTime"] = pendulum.parse(balance["modificationTime"])
+            if (
+                balance["modificationTime"] > start_time
+                and balance["modificationTime"] < end_time
+            ):
+                yield balance

{ingestr-0.13.82 → ingestr-0.13.83}/pyproject.toml

@@ -93,7 +93,8 @@ exclude = [
     'src/zoom/.*',
     'src/clickup/.*',
     'src/influxdb/.*',
-    'src/freshdesk/.*'
+    'src/freshdesk/.*',
+    'src/wise/.*'
   ]
 
 [[tool.mypy.overrides]]
@@ -125,7 +126,8 @@ module = [
   "ingestr.src.zoom.*",
   "ingestr.src.clickup.*",
   "ingestr.src.influxdb",
-  "ingestr.src.freshdesk"
+  "ingestr.src.freshdesk",
+  "ingestr.src.wise"
 ]
 follow_imports = "skip"
 

ingestr-0.13.82/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.82"