ingestr 0.13.82__py3-none-any.whl → 0.13.83__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.82"
+version = "v0.13.83"

ingestr/src/factory.py

@@ -73,6 +73,7 @@ from ingestr.src.sources import (
     StripeAnalyticsSource,
     TikTokSource,
     TrustpilotSource,
+    WiseSource,
     ZendeskSource,
     ZoomSource,
 )
@@ -192,6 +193,7 @@ class SourceDestinationFactory:
         "zoom": ZoomSource,
         "clickup": ClickupSource,
         "influxdb": InfluxDBSource,
+        "wise": WiseSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr/src/sources.py

@@ -3386,3 +3386,41 @@ class InfluxDBSource:
             start_date=start_date,
             end_date=end_date,
         ).with_resources(table)
+
+
+class WiseSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed = urlparse(uri)
+        params = parse_qs(parsed.query)
+        api_key = params.get("api_key")
+
+        if not api_key:
+            raise MissingValueError("api_key", "Wise")
+
+        if table not in ["profiles", "transfers", "balances"]:
+            raise ValueError(
+                f"Resource '{table}' is not supported for Wise source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+            )
+
+        start_date = kwargs.get("interval_start")
+        if start_date:
+            start_date = ensure_pendulum_datetime(start_date).in_timezone("UTC")
+        else:
+            start_date = pendulum.datetime(2020, 1, 1).in_timezone("UTC")
+
+        end_date = kwargs.get("interval_end")
+        if end_date:
+            end_date = ensure_pendulum_datetime(end_date).in_timezone("UTC")
+        else:
+            end_date = None
+
+        from ingestr.src.wise import wise_source
+
+        return wise_source(
+            api_key=api_key[0],
+            start_date=start_date,
+            end_date=end_date,
+        ).with_resources(table)

ingestr/src/wise/__init__.py

@@ -0,0 +1,68 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from .client import WiseClient
+
+
+@dlt.source(max_table_nesting=0)
+def wise_source(
+    api_key: str,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None = None,
+) -> Iterable[DltResource]:
+    client = WiseClient(api_key)
+
+    # List of all profiles belonging to user.
+    @dlt.resource(write_disposition="merge", name="profiles", primary_key="id")
+    def profiles() -> Iterable[TDataItem]:
+        yield from client.fetch_profiles()
+
+    # List transfers for a profile.
+    @dlt.resource(write_disposition="merge", name="transfers", primary_key="id")
+    def transfers(
+        profiles=profiles,
+        datetime=dlt.sources.incremental(
+            "created",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ):
+        if datetime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = datetime.end_value
+
+        start_dt = datetime.last_value
+
+        for profile in profiles:
+            yield from client.fetch_transfers(profile["id"], start_dt, end_dt)
+
+    # Retrieve the user's multi-currency account balance accounts. It returns all balance accounts the profile has.
+    @dlt.resource(write_disposition="merge", name="balances", primary_key="id")
+    def balances(
+        profiles=profiles,
+        datetime=dlt.sources.incremental(
+            "modificationTime",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        if datetime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = datetime.end_value
+
+        start_dt = datetime.last_value
+
+        for profile in profiles:
+            yield from client.fetch_balances(profile["id"], start_dt, end_dt)
+
+    return profiles, transfers, balances

ingestr/src/wise/client.py

@@ -0,0 +1,63 @@
+from typing import Iterable
+
+import pendulum
+from dlt.sources.helpers.requests import Client
+
+
+class WiseClient:
+    BASE_URL = "https://api.transferwise.com"
+
+    def __init__(self, api_key: str) -> None:
+        self.session = Client(raise_for_status=False).session
+        self.session.headers.update({"Authorization": f"Bearer {api_key}"})
+
+    # https://docs.wise.com/api-docs/api-reference/profile#list-profiles
+    def fetch_profiles(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v2/profiles"
+        resp = self.session.get(url)
+        resp.raise_for_status()
+        for profile in resp.json():
+            yield profile
+
+    # https://docs.wise.com/api-docs/api-reference/transfer#list-transfers
+    def fetch_transfers(
+        self, profile_id: str, start_time=pendulum.DateTime, end_time=pendulum.DateTime
+    ):
+        offset = 0
+
+        while True:
+            data = self.session.get(
+                f"{self.BASE_URL}/v1/transfers",
+                params={
+                    "profile": profile_id,
+                    "createdDateStart": start_time.to_date_string(),
+                    "createdDateEnd": end_time.to_date_string(),
+                    "limit": 100,
+                    "offset": offset,
+                },
+            )
+            response_data = data.json()
+
+            if not response_data or len(response_data) == 0:
+                break
+
+            for transfer in response_data:
+                transfer["created"] = pendulum.parse(transfer["created"])
+
+                yield transfer
+            offset += 100
+
+    # https://docs.wise.com/api-docs/api-reference/balance#list
+    def fetch_balances(
+        self, profile_id: str, start_time=pendulum.DateTime, end_time=pendulum.DateTime
+    ) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v4/profiles/{profile_id}/balances"
+        resp = self.session.get(url, params={"types": "STANDARD,SAVINGS"})
+        resp.raise_for_status()
+        for balance in resp.json():
+            balance["modificationTime"] = pendulum.parse(balance["modificationTime"])
+            if (
+                balance["modificationTime"] > start_time
+                and balance["modificationTime"] < end_time
+            ):
+                yield balance

{ingestr-0.13.82.dist-info → ingestr-0.13.83.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.82
+Version: 0.13.83
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.82.dist-info → ingestr-0.13.83.dist-info}/RECORD

@@ -2,16 +2,16 @@ ingestr/conftest.py,sha256=OE2yxeTCosS9CUFVuqNypm-2ftYvVBeeq7egm3878cI,1981
 ingestr/main.py,sha256=qoWHNcHh0-xVnyQxbQ-SKuTxPb1RNV3ENkCpqO7CLrk,26694
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
-ingestr/src/buildinfo.py,sha256=S1Cgqn9Xc0avTk0r_rJ8fMXucqD7c5O68FsKjSWj2Pc,21
+ingestr/src/buildinfo.py,sha256=MHjeovk0HwiWzmF0hXLo3Hw11MPw21fjYmBbB8wW8Fo,21
 ingestr/src/destinations.py,sha256=M2Yni6wiWcrvZ8EPJemidqxN156l0rehgCc7xuil7mo,22840
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=rF5Ry4o4t8KulSPBtrd7ZKCI_0TH1DAetG0zs9H7oik,6792
+ingestr/src/factory.py,sha256=SQr7s-WCx_jL8DmPbeCc4VuE35z1J7gCAwKGAkOpv2M,6836
 ingestr/src/filters.py,sha256=LLecXe9QkLFkFLUZ92OXNdcANr1a8edDxrflc2ko_KA,1452
 ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,732
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=Yaej1Iy75JPOccw6jYL6kbhzYFq33t4YelGT-NiAA5I,119661
+ingestr/src/sources.py,sha256=NfD7pj0WOD1wQOV3oZkHmGOAwHMuCCO_Mzf8F8EpobU,120918
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -134,6 +134,8 @@ ingestr/src/tiktok_ads/__init__.py,sha256=_upvSv79uktELTzg78VavYgQftfhp4YM6OXfpk
 ingestr/src/tiktok_ads/tiktok_helpers.py,sha256=jmWHvZzN1Vt_PWrJkgq5a2wIwon-OBEzXoZx0jEy-74,3905
 ingestr/src/trustpilot/__init__.py,sha256=ofhjep4qRPIi8q41qc97QVex8UbWF-Fd7gUsqeQlQX8,1279
 ingestr/src/trustpilot/client.py,sha256=zKYt5C7nrR83Id0KN49EPmtml8MEtlSPlAosEFU3VXY,1616
+ingestr/src/wise/__init__.py,sha256=EiXWE89Jb2Do1t8HK1-GZRS2KwgaiThxsHVEpaBcoh4,2087
+ingestr/src/wise/client.py,sha256=2hTCuOB24-HiwTp02o5i99KF5rjI0CXqgIb9V6gAjjE,2247
 ingestr/src/zendesk/__init__.py,sha256=tmJ_jdb6kpwmEKpcv6Im71-bOZI6h-Tcofe18OH4I24,17762
 ingestr/src/zendesk/settings.py,sha256=Vdj706nTJFQ-3KH4nO97iYCQuba3dV3E9gfnmLK6xwU,2294
 ingestr/src/zendesk/helpers/__init__.py,sha256=YTJejCiUjfIcsj9FrkY0l-JGYDI7RRte1Ydq5FDH_0c,888
@@ -151,8 +153,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
-ingestr-0.13.82.dist-info/METADATA,sha256=4edZaTP7apukcZRUY4J00G99O3d7Qxz37csXXCNhaww,15182
-ingestr-0.13.82.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.82.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.82.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.82.dist-info/RECORD,,
+ingestr-0.13.83.dist-info/METADATA,sha256=SP8vz_9PH-RBNLkM7QzR3YEMIR8nxXhsGyJN9s09Cig,15182
+ingestr-0.13.83.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.83.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.83.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.83.dist-info/RECORD,,